Merge branch 'lmenezes'

# Conflicts:
#	public/overview.html

.travis.yml

@@ -4,7 +4,7 @@ scala:
   - 2.12.8
 
 jdk:
-  - oraclejdk8
+  - openjdk8
   - openjdk11
 
 script: sbt ++$TRAVIS_SCALA_VERSION test

CHANGES.md

@@ -275,3 +275,46 @@ First cerebro release.
 
 - Support user-attr-template for LDAP auth
 - Allow forwarding proxy headers to ES
+
+### v0.8.5 - September 18th, 2019
+
+#### Enhancements
+
+- Update json-tree to 0.3.0 #405
+
+#### Bug fixes
+
+- Read closed indices from routing table
+- Use full setting name for index settings updates #382
+- Use openjdk8 instead of oraclejdk8 for travis testing
+- Update json-tree to 0.3.0 #405
+
+### v0.9.0 - April 24th, 2020
+
+#### Enhancements
+
+- GET as default verb in the REST console #437
+- Upgrade to the latest Play and other core libraries #439
+- Added possibility of configuring port using CEREBRO_PORT environment var. #438
+- Allow selecting kind of shard allocation in overview #423
+- Sort indices on snapshot restore (5317ddd54de574708dbf02e713b5e2d0865441e0). Closes #419
+- Change login user input tooltip (66b73ddd5ab818534760bc64895899825a20ec62). Closes #424
+- Add content-type to copied curl command (e93ab9948c3c650a9fcdd5be2a7edbe1976cab05). Closes #426
+- Include aliases in rest autocompletion (abb8ab73992dfe9708b2b34f8ba2b75924d4d7ec)
+- Drop support for deprecated endpoints.
+- Drop type support on rest autocomplete. 
+
+#### Bug fixes
+
+- Tolerate missing settings for repositories (2bed3327b056295b215e6a809dcbf50e8b0e7926). Closes #409
+- Handle post typeless indices mappings on analysis (9dbece339a014971307506977ec71045e977d576). Closes #412
+
+### v0.9.1 - May 20th, 2020
+
+#### Enhancements
+- Display node attributes on overview / nodes view
+
+### v0.9.2 - June 18th, 2020
+
+#### Bug fixes
+- Handle node info for nodes without defined attributes Closes #448

Gruntfile.js

@@ -84,6 +84,7 @@ module.exports = function(grunt) {
       ],
       options: {
         preset: 'google',
+        maximumLineLength: 120,
         requireCamelCaseOrUpperCaseIdentifiers: "ignoreProperties"
       }
     }

README.md

@@ -82,6 +82,7 @@ You can the pass this file as argument using:
  docker run -p 9000:9000 --env-file env-ldap  lmenezes/cerebro
 ```
 
+There are some examples of configuration in the [examples folder](./examples).
 
 #### Other settings
 

app/controllers/BaseController.scala

@@ -5,7 +5,7 @@ import exceptions.MissingRequiredParamException
 import models.{CerebroRequest, CerebroResponse, Hosts}
 import play.api.Logger
 import play.api.libs.json._
-import play.api.mvc.{Controller, InjectedController, Result}
+import play.api.mvc.{InjectedController, Result}
 import services.exception.RequestFailedException
 
 import scala.concurrent.ExecutionContext.Implicits.global

app/controllers/ClusterOverviewController.scala

@@ -21,7 +21,8 @@ class ClusterOverviewController @Inject()(val authentication: AuthenticationModu
   }
 
   def disableShardAllocation = process { request =>
-    client.disableShardAllocation(request.target).map { response =>
+    val kind = request.get("kind")
+    client.disableShardAllocation(request.target, kind).map { response =>
       CerebroResponse(response.status, response.body)
     }
   }

app/controllers/RestController.scala

@@ -2,12 +2,13 @@ package controllers
 
 import java.sql.Date
 import java.text.SimpleDateFormat
-import javax.inject.Inject
 
+import javax.inject.Inject
 import controllers.auth.AuthenticationModule
 import dao.{DAOException, RestHistoryDAO, RestRequest}
 import elastic.{ElasticClient, Error, Success}
-import models.{CerebroResponse, ClusterMapping, Hosts}
+import models.rest.AutocompletionIndices
+import models.{CerebroResponse, Hosts}
 import play.api.libs.json.{JsArray, JsString, Json}
 
 import scala.concurrent.ExecutionContext.Implicits.global
@@ -43,10 +44,10 @@ class RestController @Inject()(val authentication: AuthenticationModule,
   }
 
   def index = process { request =>
-    client.getClusterMapping(request.target).map {
+    client.getAliases(request.target).map {
       case Success(status, body) =>
         val data = Json.obj(
-          "mappings" -> ClusterMapping(body),
+          "indices" -> AutocompletionIndices(body),
           "host"    -> request.target.host.name
         )
         CerebroResponse(status, data)

app/controllers/auth/ldap/LDAPAuthConfig.scala

@@ -11,14 +11,16 @@ class LDAPAuthConfig(config: Configuration) extends AuthConfig {
   final val method = getSetting("method")
   final val url = getSetting("url")
   final val baseDN = getSetting("base-dn")
-  final val bindDN = getSetting("bind-dn")
-  final val bindPwd = getSetting("bind-pw")
 
 
   final val groupMembership: Option[LDAPGroupSearchConfig] = {
+    val bindDN = getSetting("bind-dn")
+    val bindPwd = getSetting("bind-pw")
     val groupAuthConfig = config.get[Configuration]("group-search")
     groupAuthConfig.getOptional[String]("group").map { group =>
       LDAPGroupSearchConfig(
+        bindDN,
+        bindPwd,
         groupAuthConfig.getOptional[String]("base-dn").getOrElse(baseDN),
         getSetting("user-attr")(groupAuthConfig),
         groupAuthConfig.getOptional[String]("user-attr-template").getOrElse(userTemplate),
@@ -28,4 +30,4 @@ class LDAPAuthConfig(config: Configuration) extends AuthConfig {
   }
 }
 
-case class LDAPGroupSearchConfig(baseDN: String, userAttr: String, userAttrTemplate:String, group: String)
+case class LDAPGroupSearchConfig(bindDN: String, bindPwd: String, baseDN: String, userAttr: String, userAttrTemplate:String, group: String)

app/controllers/auth/ldap/LDAPAuthService.scala

@@ -37,8 +37,8 @@ class LDAPAuthService @Inject()(globalConfig: Configuration) extends AuthService
 
   def checkGroupMembership(username: String, groupConfig: LDAPGroupSearchConfig): Boolean = {
     val props = new Hashtable[String, String]()
-    props.put(Context.SECURITY_PRINCIPAL, config.bindDN)
-    props.put(Context.SECURITY_CREDENTIALS, config.bindPwd)
+    props.put(Context.SECURITY_PRINCIPAL, groupConfig.bindDN)
+    props.put(Context.SECURITY_CREDENTIALS, groupConfig.bindPwd)
     props.put(Context.REFERRAL, "follow")
     val user     = groupConfig.userAttrTemplate.format(username, config.baseDN)
     val controls = new SearchControls()

app/elastic/ElasticClient.scala

@@ -56,7 +56,7 @@ trait ElasticClient {
 
   def enableShardAllocation(target: ElasticServer): Future[ElasticResponse]
 
-  def disableShardAllocation(target: ElasticServer): Future[ElasticResponse]
+  def disableShardAllocation(target: ElasticServer, kind: String): Future[ElasticResponse]
 
   def getShardStats(index: String, target: ElasticServer): Future[ElasticResponse]
 

app/elastic/HTTPElasticClient.scala

@@ -124,8 +124,8 @@ class HTTPElasticClient @Inject()(client: WSClient) extends ElasticClient {
   def enableShardAllocation(target: ElasticServer) =
     putClusterSettings(allocationSettings("all"), target)
 
-  def disableShardAllocation(target: ElasticServer) =
-    putClusterSettings(allocationSettings("none"), target)
+  def disableShardAllocation(target: ElasticServer, kind: String) =
+    putClusterSettings(allocationSettings(kind), target)
 
   def getShardStats(index: String, target: ElasticServer) = {
     val path = s"/${encoded(index)}/_stats?level=shards&human=true"

app/models/analysis/IndexFields.scala

@@ -14,10 +14,15 @@ object IndexFields {
 
   def apply(index: String, data: JsValue) = {
     val docTypes = (data \ index \ "mappings").as[JsObject].keys
-    val fields = docTypes.flatMap { docType =>
-      extractProperties((data \ index \ "mappings" \ docType \ "properties").as[JsValue])
-    }.toSeq
-    JsArray(fields.map(JsString(_)))
+    val fields = if (docTypes.size == 1 && docTypes.head == "properties") {
+      extractProperties((data \ index \ "mappings" \ "properties").as[JsValue])
+    } else { // FIXME: ES < 7
+      docTypes.flatMap { docType =>
+        extractProperties((data \ index \ "mappings" \ docType \ "properties").as[JsValue])
+      }.toSeq
+    }
+
+    JsArray(fields.map(JsString))
   }
 
   def extractProperties(data: JsValue): Seq[String] = {

app/models/commons/NodeInfo.scala

@@ -0,0 +1,23 @@
+package models.commons
+
+import play.api.libs.json.{JsObject, JsValue, Json}
+
+trait NodeInfo {
+
+  private val InternalNodeAttributes = Seq(
+    "ml.machine_memory",
+    "xpack.installed",
+    "transform.node",
+    "ml.max_open_jobs"
+  )
+
+  def attrs(info: JsValue) = {
+    val map =
+      (info \ "attributes").asOpt[JsObject].map(_.value.filterNot {
+        case (attr, _) => InternalNodeAttributes.contains(attr)
+      }).getOrElse(Map())
+
+    JsObject(map.toSeq)
+  }
+
+}

app/models/nodes/Node.scala

@@ -1,9 +1,9 @@
 package models.nodes
 
-import models.commons.NodeRoles
+import models.commons.{NodeInfo, NodeRoles}
 import play.api.libs.json._
 
-object Node {
+object Node extends NodeInfo {
 
   def apply(id: String, currentMaster: Boolean, info: JsValue, stats: JsValue): JsValue = {
     val jvmVersion = (info \ "jvm" \ "version").asOpt[JsString].getOrElse(JsNull)
@@ -18,6 +18,7 @@ object Node {
       "cpu" -> cpu(stats),
       "uptime" -> (stats \ "jvm" \ "uptime_in_millis").as[JsValue],
       "jvm" -> jvmVersion,
+      "attributes" -> attrs(info),
       "version" -> (info \ "version").as[JsValue]
     ) ++ roles(info)
   }

app/models/overview/ClusterOverview.scala

@@ -49,10 +49,13 @@ object ClusterOverview {
   def buildIndices(clusterState: JsValue, indicesStats: JsValue, aliases: JsValue): Seq[JsValue] = {
     val routingTable = (clusterState \ "routing_table" \ "indices").as[JsObject].value
     val blocks = (clusterState \ "blocks" \ "indices").asOpt[JsObject].getOrElse(Json.obj())
+    val stats = (indicesStats \ "indices").asOpt[JsObject].getOrElse(Json.obj())
     val indices = routingTable.map { case (index, shards) =>
-      val indexStats   = (indicesStats \ "indices" \ index).asOpt[JsObject].getOrElse(Json.obj())
+      // Since stats and blocks are JsObject objects potentially big, it's checked that key exists in that object.
+      // This way, it avoids building a JsUndefined instance with a big string as explained in #467
+      val indexStats = if (stats.value contains index) (stats \ index).asOpt[JsObject].getOrElse(Json.obj()) else Json.obj()
+      val indexBlock = if (blocks.value contains index ) (blocks \ index).asOpt[JsObject].getOrElse(Json.obj()) else Json.obj()
       val indexAliases = (aliases \ index \ "aliases").asOpt[JsObject].getOrElse(Json.obj()) // 1.4 < does not return aliases obj
-      val indexBlock = (blocks \ index).asOpt[JsObject].getOrElse(Json.obj())
       Index(index, indexStats, shards, indexAliases, indexBlock)
     }.toSeq
 

app/models/overview/Node.scala

@@ -1,9 +1,9 @@
 package models.overview
 
-import models.commons.NodeRoles
+import models.commons.{NodeInfo, NodeRoles}
 import play.api.libs.json._
 
-object Node {
+object Node extends NodeInfo {
 
   def apply(id: String, info: JsValue, stats: JsValue, masterNodeId: String) = {
     val nodeRoles = NodeRoles(stats)
@@ -35,7 +35,8 @@ object Node {
         "used_percent" -> (stats \ "jvm" \ "mem" \ "heap_used_percent").as[JsNumber],
         "max" -> (stats \ "jvm" \ "mem" \ "heap_max_in_bytes").as[JsNumber]
       ),
-      "disk" -> disk(stats)
+      "disk" -> disk(stats),
+      "attributes" -> attrs(info)
     )
   }
 

app/models/repository/Repositories.scala

@@ -13,7 +13,7 @@ object Repositories {
             Json.obj(
               "name" -> name,
               "type" -> (info \ "type").as[JsValue],
-              "settings" -> (info \ "settings").as[JsValue]
+              "settings" -> (info \ "settings").asOpt[JsObject].getOrElse[JsObject](Json.obj())
             )
           }.toSeq
         )

app/models/rest/AutocompletionIndices.scala

@@ -0,0 +1,14 @@
+package models.rest
+
+import play.api.libs.json.{JsArray, JsObject, JsString, JsValue}
+
+object AutocompletionIndices {
+
+  def apply(aliases: JsValue): JsArray =
+    JsArray(
+      aliases.as[JsObject].value.flatMap { case (idx, data) =>
+        (data \ "aliases").as[JsObject].keys + idx
+      }.toSeq.distinct.map(JsString)
+    )
+
+}

app/views/auth/login.scala.html

@@ -39,7 +39,7 @@ <h4>Cerebro
           <form class="form-signin" action="auth/login" method="POST">
             <div class="form-group">
               <label for="inputUser" class="sr-only">User</label>
-              <input type="text" name="user" id="inputUser" class="form-control form-control-sm" placeholder="LDAP user" required autofocus>
+              <input type="text" name="user" id="inputUser" class="form-control form-control-sm" placeholder="User" required autofocus>
             </div>
             <div class="form-group">
               <label for="inputPassword" class="sr-only">Password</label>

build.sbt

@@ -7,9 +7,9 @@ packageSummary := "Elasticsearch web admin tool"
 packageDescription := """cerebro is an open source(MIT License) elasticsearch web admin tool built
   using Scala, Play Framework, AngularJS and Bootstrap."""
 
-version := "0.8.4"
+version := "0.9.2"
 
-scalaVersion := "2.12.8"
+scalaVersion := "2.12.11"
 
 rpmVendor := "lmenezes"
 
@@ -18,10 +18,10 @@ rpmLicense := Some("MIT")
 rpmUrl := Some("http://github.com/lmenezes/cerebro")
 
 libraryDependencies ++= Seq(
-  "com.typesafe.play" %% "play"                    % "2.7.0",
-  "com.typesafe.play" %% "play-json"               % "2.7.2",
-  "com.typesafe.play" %% "play-slick"              % "4.0.0",
-  "com.typesafe.play" %% "play-slick-evolutions"   % "4.0.0",
+  "com.typesafe.play" %% "play"                    % "2.8.1",
+  "com.typesafe.play" %% "play-json"               % "2.8.1",
+  "com.typesafe.play" %% "play-slick"              % "5.0.0",
+  "com.typesafe.play" %% "play-slick-evolutions"   % "5.0.0",
   "org.xerial"        %  "sqlite-jdbc"             % "3.23.1",
   "org.specs2"        %% "specs2-junit"  % "4.3.4" % "test",
   "org.specs2"        %% "specs2-core"   % "4.3.4" % "test",

conf/application.conf

@@ -17,6 +17,11 @@ rest.history.size = 50 // defaults to 50 if not specified
 #data.path: "/var/lib/cerebro/cerebro.db"
 data.path = "./cerebro.db"
 
+play {
+  # Cerebro port, by default it's 9000 (play's default)
+  server.http.port = ${?CEREBRO_PORT}
+}
+
 es = {
   gzip = true
 }

examples/basic_auth/README.md

@@ -0,0 +1,19 @@
+# How to configure basic auth
+
+## Dockerized environment
+
+When running cerebro's docker image the following environment variable must be set.
+
+- `AUTH_TYPE`: must be set to `basic`  
+- `BASIC_AUTH_USER`
+- `BASIC_AUTH_PWD`
+
+This [docker-compose](./docker-compose.yml) shows how to configure cerebro with basic auth and the user `admin`/`admin`.
+
+## Downloading the tgz
+
+In order to configure basic auth with `admin`/`admin` user just run the following command:
+
+`./bin/cerebro -DAUTH_TYPE=basic -DBASIC_AUTH_USER=admin -DBASIC_AUTH_PWD=admin`
+
+If overriding more settings is needed, it might be more convenient to create your `application.conf` file as explainded [here](https://github.com/lmenezes/cerebro#other-settings)

examples/basic_auth/docker-compose.yml

@@ -0,0 +1,11 @@
+version: '3'
+services:
+  cerebro:
+    image: lmenezes/cerebro
+    environment:
+      # Basic auth using admin/admin user
+      - AUTH_TYPE=basic
+      - BASIC_AUTH_USER=admin
+      - BASIC_AUTH_PWD=admin
+    ports:
+      - 9000:9000