Add indexing complete

.gitignore

@@ -22,3 +22,6 @@ server.pid
 activator-*.sbt
 node_modules
 cerebro.db
+
+# Ignore metals
+metals.sbt

app/models/overview/ClosedIndex.scala

@@ -1,14 +1,15 @@
 package models.overview
 
-import play.api.libs.json.{JsBoolean, JsString, Json}
+import play.api.libs.json.{JsBoolean, JsFalse, JsString, Json}
 
 object ClosedIndex {
 
   def apply(name: String) =
     Json.obj(
       "name" -> JsString(name),
       "closed" -> JsBoolean(true),
-      "special" -> JsBoolean(name.startsWith("."))
+      "special" -> JsBoolean(name.startsWith(".")),
+      "complete" -> JsFalse
     )
 
 }

app/models/overview/ClusterOverview.scala

@@ -1,13 +1,15 @@
 package models.overview
 
+import java.lang.{ Boolean => JBoolean }
+
 import play.api.libs.json._
 
 object ClusterOverview {
 
   def apply(clusterState: JsValue, nodesStats: JsValue, indicesStats: JsValue,
             clusterSettings: JsValue, aliases: JsValue, clusterHealth: JsValue,
-            nodesInfo: JsValue): JsValue = {
-    val indices = buildIndices(clusterState, indicesStats, aliases)
+            nodesInfo: JsValue, indexingCompleteSettings: JsValue): JsValue = {
+    val indices = buildIndices(clusterState, indicesStats, aliases, indexingCompleteSettings)
 
     val masterNodeId = (clusterState \ "master_node").as[String]
 
@@ -31,6 +33,7 @@ object ClusterOverview {
       "total_indices" -> JsNumber(indices.size),
       "closed_indices" -> JsNumber(indices.count { idx => (idx \ "closed").as[Boolean] }),
       "special_indices" -> JsNumber(indices.count { idx => (idx \ "special").as[Boolean] }),
+      "complete_indices" -> JsNumber(indices.count { idx => (idx \ "complete").as[Boolean] }),
       "indices" -> JsArray(indices),
       "nodes" -> buildNodes(masterNodeId, nodesInfo, nodesStats),
       "shard_allocation" -> JsBoolean(shardAllocation)
@@ -46,17 +49,20 @@ object ClusterOverview {
       }.toSeq
     )
 
-  def buildIndices(clusterState: JsValue, indicesStats: JsValue, aliases: JsValue): Seq[JsValue] = {
+  def buildIndices(clusterState: JsValue, indicesStats: JsValue, aliases: JsValue, indexingCompleteSettings: JsValue): Seq[JsValue] = {
     val routingTable = (clusterState \ "routing_table" \ "indices").as[JsObject].value
     val blocks = (clusterState \ "blocks" \ "indices").asOpt[JsObject].getOrElse(Json.obj())
     val stats = (indicesStats \ "indices").asOpt[JsObject].getOrElse(Json.obj())
+    val completed = indexingCompleteSettings.as[JsObject].value.collect {
+      case (idx, value) if (value \\ "indexing_complete").map(ic => JBoolean.parseBoolean(ic.as[JsString].value)).exists(_ == true) => idx
+    }.toSet
     val indices = routingTable.map { case (index, shards) =>
       // Since stats and blocks are JsObject objects potentially big, it's checked that key exists in that object.
       // This way, it avoids building a JsUndefined instance with a big string as explained in #467
       val indexStats = if (stats.value contains index) (stats \ index).asOpt[JsObject].getOrElse(Json.obj()) else Json.obj()
       val indexBlock = if (blocks.value contains index ) (blocks \ index).asOpt[JsObject].getOrElse(Json.obj()) else Json.obj()
       val indexAliases = (aliases \ index \ "aliases").asOpt[JsObject].getOrElse(Json.obj()) // 1.4 < does not return aliases obj
-      Index(index, indexStats, shards, indexAliases, indexBlock)
+      Index(index, indexStats, shards, indexAliases, indexBlock, if (completed(index)) JsTrue else JsFalse)
     }.toSeq
 
     val closedIndices = blocks.value.collect { // ES < 7.X does not return routing_table for closed indices

app/models/overview/Index.scala

@@ -4,13 +4,14 @@ import play.api.libs.json._
 
 object Index {
 
-  def apply(name: String, stats: JsValue, routingTable: JsValue, aliases: JsValue, indexBlock: JsObject): JsValue = {
+  def apply(name: String, stats: JsValue, routingTable: JsValue, aliases: JsValue, indexBlock: JsObject, indexingComplete: JsBoolean): JsValue = {
     val shardMap = createShardMap(routingTable)
 
     JsObject(Seq(
       "name" -> JsString(name),
       "closed" -> isClosed(indexBlock),
       "special" -> JsBoolean(name.startsWith(".")),
+      "complete" -> indexingComplete,
       "unhealthy" -> JsBoolean(isIndexUnhealthy(shardMap)),
       "doc_count" -> (stats \ "primaries" \ "docs" \ "count").asOpt[JsNumber].getOrElse(JsNumber(0)),
       "deleted_docs" -> (stats \ "primaries" \ "docs" \ "deleted").asOpt[JsNumber].getOrElse(JsNumber(0)),

app/services/overview/OverviewDataService.scala

@@ -20,7 +20,8 @@ class OverviewDataService @Inject()(client: ElasticClient) {
       "_cluster/settings",
       "_aliases",
       "_cluster/health",
-      s"_nodes/_all/os,jvm?human=true"
+      "_nodes/_all/os,jvm?human=true",
+      "_all/_settings/index.lifecycle.indexing_complete"
     )
     Future.sequence(apis.map(client.executeRequest("GET", _, None, target))).map { responses =>
       responses.zipWithIndex.find(_._1.isInstanceOf[Error]) match {
@@ -35,7 +36,8 @@ class OverviewDataService @Inject()(client: ElasticClient) {
             responses(3).body, // cluster settings
             responses(4).body, // aliases
             responses(5).body, // cluster health
-            responses(6).body  // nodes
+            responses(6).body, // nodes
+            responses(7).body  // indexing_complete
           )
       }
     }

public/js/app.js

@@ -1136,9 +1136,11 @@ angular.module('cerebro').controller('OverviewController', ['$scope', '$http',
     $scope.initializing_shards = 0;
     $scope.closed_indices = 0;
     $scope.special_indices = 0;
+    $scope.complete_indices = 0;
     $scope.shardAllocation = true;
 
-    $scope.indices_filter = new IndexFilter('', false, false, true, true, 0);
+    $scope.indices_filter =
+      new IndexFilter('', false, false, false, true, true, 0);
     $scope.nodes_filter = new NodeFilter('', true, false, false, false, 0);
 
     $scope.getPageSize = function() {
@@ -1181,6 +1183,7 @@ angular.module('cerebro').controller('OverviewController', ['$scope', '$http',
           $scope.initializing_shards = data.initializing_shards;
           $scope.closed_indices = data.closed_indices;
           $scope.special_indices = data.special_indices;
+          $scope.complete_indices = data.complete_indices;
           $scope.shardAllocation = data.shard_allocation;
           if (!$scope.unassigned_shards &&
             !$scope.relocating_shards &&
@@ -1198,6 +1201,7 @@ angular.module('cerebro').controller('OverviewController', ['$scope', '$http',
           $scope.initializing_shards = 0;
           $scope.closed_indices = 0;
           $scope.special_indices = 0;
+          $scope.complete_indices = 0;
           $scope.shardAllocation = true;
         }
       );
@@ -2310,10 +2314,11 @@ function GroupedSettings(settings) {
   this.groups = Object.values(groups);
 }
 
-function IndexFilter(name, closed, special, healthy, asc, timestamp) {
+function IndexFilter(name, closed, special, complete, healthy, asc, timestamp) {
   this.name = name;
   this.closed = closed;
   this.special = special;
+  this.complete = complete;
   this.healthy = healthy;
   this.sort = 'name';
   this.asc = asc;
@@ -2340,6 +2345,7 @@ function IndexFilter(name, closed, special, healthy, asc, timestamp) {
       this.name,
       this.closed,
       this.special,
+      this.complete,
       this.healthy,
       this.asc,
       this.timestamp
@@ -2352,6 +2358,7 @@ function IndexFilter(name, closed, special, healthy, asc, timestamp) {
       this.name === other.name &&
       this.closed === other.closed &&
       this.special === other.special &&
+      this.complete === other.complete &&
       this.healthy === other.healthy &&
       this.asc === other.asc &&
       this.timestamp === other.timestamp
@@ -2363,13 +2370,17 @@ function IndexFilter(name, closed, special, healthy, asc, timestamp) {
       !this.name &&
       this.closed &&
       this.special &&
+      this,complete &&
       this.healthy &&
       this.asc
     );
   };
 
   this.matches = function(index) {
     var matches = true;
+    if (!this.complete && index.complete) {
+      matches = false;
+    }
     if (!this.special && index.special) {
       matches = false;
     }

public/overview.html

@@ -8,20 +8,27 @@
                  ng-model-options='{ debounce: 250 }'
                  placeholder="filter indices by name or aliases">
         </div>
-        <div class="col-lg-3 col-sm-3 col-xs-6 form-group">
+        <div class="col-lg-2 col-sm-2 col-xs-4 form-group">
           <div class="checkbox">
             <label>
               <input type="checkbox" ng-model="paginator.filter.closed"> closed ({{closed_indices}})
             </label>
           </div>
         </div>
-        <div class="col-lg-3 col-sm-3 col-xs-6 form-group">
+        <div class="col-lg-2 col-sm-2 col-xs-4 form-group">
           <div class="checkbox">
             <label>
               <input type="checkbox" ng-model="paginator.filter.special"> .special ({{special_indices}})
             </label>
           </div>
         </div>
+        <div class="col-lg-2 col-sm-2 col-xs-4 form-group">
+          <div class="checkbox">
+            <label>
+              <input type="checkbox" ng-model="paginator.filter.complete"> complete ({{complete_indices}})
+            </label>
+          </div>
+        </div>
       </div>
     </div>
     <div class="col-lg-4">

src/app/components/overview/controller.js

@@ -13,9 +13,11 @@ angular.module('cerebro').controller('OverviewController', ['$scope', '$http',
     $scope.initializing_shards = 0;
     $scope.closed_indices = 0;
     $scope.special_indices = 0;
+    $scope.complete_indices = 0;
     $scope.shardAllocation = true;
 
-    $scope.indices_filter = new IndexFilter('', false, false, true, true, 0);
+    $scope.indices_filter =
+      new IndexFilter('', false, false, false, true, true, 0);
     $scope.nodes_filter = new NodeFilter('', true, false, false, false, 0);
 
     $scope.getPageSize = function() {
@@ -58,6 +60,7 @@ angular.module('cerebro').controller('OverviewController', ['$scope', '$http',
           $scope.initializing_shards = data.initializing_shards;
           $scope.closed_indices = data.closed_indices;
           $scope.special_indices = data.special_indices;
+          $scope.complete_indices = data.complete_indices;
           $scope.shardAllocation = data.shard_allocation;
           if (!$scope.unassigned_shards &&
             !$scope.relocating_shards &&
@@ -75,6 +78,7 @@ angular.module('cerebro').controller('OverviewController', ['$scope', '$http',
           $scope.initializing_shards = 0;
           $scope.closed_indices = 0;
           $scope.special_indices = 0;
+          $scope.complete_indices = 0;
           $scope.shardAllocation = true;
         }
       );

src/app/shared/index_filter.js

@@ -1,7 +1,8 @@
-function IndexFilter(name, closed, special, healthy, asc, timestamp) {
+function IndexFilter(name, closed, special, complete, healthy, asc, timestamp) {
   this.name = name;
   this.closed = closed;
   this.special = special;
+  this.complete = complete;
   this.healthy = healthy;
   this.sort = 'name';
   this.asc = asc;
@@ -28,6 +29,7 @@ function IndexFilter(name, closed, special, healthy, asc, timestamp) {
       this.name,
       this.closed,
       this.special,
+      this.complete,
       this.healthy,
       this.asc,
       this.timestamp
@@ -40,6 +42,7 @@ function IndexFilter(name, closed, special, healthy, asc, timestamp) {
       this.name === other.name &&
       this.closed === other.closed &&
       this.special === other.special &&
+      this.complete === other.complete &&
       this.healthy === other.healthy &&
       this.asc === other.asc &&
       this.timestamp === other.timestamp
@@ -51,13 +54,17 @@ function IndexFilter(name, closed, special, healthy, asc, timestamp) {
       !this.name &&
       this.closed &&
       this.special &&
+      this,complete &&
       this.healthy &&
       this.asc
     );
   };
 
   this.matches = function(index) {
     var matches = true;
+    if (!this.complete && index.complete) {
+      matches = false;
+    }
     if (!this.special && index.special) {
       matches = false;
     }

test/models/overview/ClusterInitializingShards.scala

@@ -4,7 +4,7 @@ import play.api.libs.json.Json
 
 object ClusterInitializingShards {
 
-  def apply() = ClusterOverview(clusterState, nodesStats, indicesStats, clusterSettings, aliases, clusterHealth, nodes)
+  def apply() = ClusterOverview(clusterState, nodesStats, indicesStats, clusterSettings, aliases, clusterHealth, nodes, indexingComplete)
 
   val clusterState = Json.parse(
     """
@@ -577,4 +577,5 @@ object ClusterInitializingShards {
     """.stripMargin
   )
 
+  val indexingComplete = Json.obj()
 }

test/models/overview/ClusterRelocatingShards.scala

@@ -709,6 +709,8 @@ object ClusterRelocatingShards extends ClusterStub {
     """.stripMargin
   )
 
+  val indexingComplete = Json.obj()
+
   val main = Json.parse(
     """
       |{

test/models/overview/ClusterStub.scala

@@ -4,7 +4,7 @@ import play.api.libs.json.JsValue
 
 trait ClusterStub {
 
-  def apply() = ClusterOverview(clusterState, nodesStats, indicesStats, clusterSettings, aliases, clusterHealth, nodes)
+  def apply() = ClusterOverview(clusterState, nodesStats, indicesStats, clusterSettings, aliases, clusterHealth, nodes, indexingComplete)
 
   val clusterState: JsValue
 
@@ -20,4 +20,5 @@ trait ClusterStub {
 
   val nodes: JsValue
 
+  val indexingComplete: JsValue
 }