ListProjects: re-implement using secondary index

The GWT UI and other parts of Gerrit still rely on the in-memory
cache for rendering the project list.
This is the first step that moves some use-cases to the QueryProjects
engine: full list without filters and showing only the active and readonly
projects.

All other existing use-cases are still based on the in-memory
cache and are going to be addressed in the follow-up of this change.

With regards to filtering by project name substring, it is not
implemented on top of the secondary index because of Issue 10446.

Bug: Issue 10380
Change-Id: I8effed5f75bdf353d9b23a3d349009e5f0535186

Documentation/access-control.txt

@@ -1354,7 +1354,7 @@ any of their groups is used.
 
 This limit applies not only to the link:cmd-query.html[`gerrit query`]
 command, but also to the web UI results pagination size in the new
-PolyGerrit UI.
+PolyGerrit UI and, limited to the full project list, in the old GWT UI.
 
 
 [[capability_readAs]]

java/com/google/gerrit/server/restapi/project/ListProjects.java

@@ -14,11 +14,15 @@
 
 package com.google.gerrit.server.restapi.project;
 
+import static com.google.common.base.Strings.emptyToNull;
+import static com.google.common.base.Strings.isNullOrEmpty;
+import static com.google.common.collect.Ordering.natural;
 import static com.google.gerrit.extensions.client.ProjectState.HIDDEN;
 import static java.nio.charset.StandardCharsets.UTF_8;
 
-import com.google.common.base.Strings;
+import com.google.common.base.Joiner;
 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSortedMap;
 import com.google.common.collect.Iterables;
 import com.google.common.flogger.FluentLogger;
 import com.google.gerrit.common.Nullable;
@@ -29,6 +33,7 @@ import com.google.gerrit.extensions.common.WebLinkInfo;
 import com.google.gerrit.extensions.restapi.AuthException;
 import com.google.gerrit.extensions.restapi.BadRequestException;
 import com.google.gerrit.extensions.restapi.BinaryResult;
+import com.google.gerrit.extensions.restapi.MethodNotAllowedException;
 import com.google.gerrit.extensions.restapi.RestReadView;
 import com.google.gerrit.extensions.restapi.TopLevelResource;
 import com.google.gerrit.extensions.restapi.Url;
@@ -51,7 +56,9 @@ import com.google.gerrit.server.project.ProjectCache;
 import com.google.gerrit.server.project.ProjectState;
 import com.google.gerrit.server.util.TreeFormatter;
 import com.google.gson.reflect.TypeToken;
+import com.google.gwtorm.server.OrmException;
 import com.google.inject.Inject;
+import com.google.inject.Provider;
 import java.io.BufferedWriter;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
@@ -67,6 +74,7 @@ import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Optional;
 import java.util.SortedMap;
 import java.util.SortedSet;
 import java.util.TreeMap;
@@ -255,6 +263,7 @@ public class ListProjects implements RestReadView<TopLevelResource> {
   private String matchSubstring;
   private String matchRegex;
   private AccountGroup.UUID groupUuid;
+  private final Provider<QueryProjects> queryProjectsProvider;
 
   @Inject
   protected ListProjects(
@@ -265,7 +274,8 @@ public class ListProjects implements RestReadView<TopLevelResource> {
       GitRepositoryManager repoManager,
       PermissionBackend permissionBackend,
       ProjectNode.Factory projectNodeFactory,
-      WebLinks webLinks) {
+      WebLinks webLinks,
+      Provider<QueryProjects> queryProjectsProvider) {
     this.currentUser = currentUser;
     this.projectCache = projectCache;
     this.groupResolver = groupResolver;
@@ -274,6 +284,7 @@ public class ListProjects implements RestReadView<TopLevelResource> {
     this.permissionBackend = permissionBackend;
     this.projectNodeFactory = projectNodeFactory;
     this.webLinks = webLinks;
+    this.queryProjectsProvider = queryProjectsProvider;
   }
 
   public List<String> getShowBranch() {
@@ -312,10 +323,62 @@ public class ListProjects implements RestReadView<TopLevelResource> {
 
   public SortedMap<String, ProjectInfo> apply()
       throws BadRequestException, PermissionBackendException {
+    Optional<String> projectQuery = expressAsProjectsQuery();
+    if (projectQuery.isPresent()) {
+      return applyAsQuery(projectQuery.get());
+    }
+
     format = OutputFormat.JSON;
     return display(null);
   }
 
+  private Optional<String> expressAsProjectsQuery() {
+    return !all
+            && state != HIDDEN
+            && isNullOrEmpty(matchPrefix)
+            && isNullOrEmpty(matchRegex)
+            && isNullOrEmpty(matchSubstring) // TODO: see Issue 10446
+            && type == FilterType.ALL
+            && showBranch.isEmpty()
+        ? Optional.of(stateToQuery())
+        : Optional.empty();
+  }
+
+  private String stateToQuery() {
+    List<String> queries = new ArrayList<>();
+    if (state == null) {
+      queries.add("(state:active OR state:read-only)");
+    } else {
+      queries.add(String.format("(state:%s)", state.name()));
+    }
+
+    return Joiner.on(" AND ").join(queries).toString();
+  }
+
+  private SortedMap<String, ProjectInfo> applyAsQuery(String query) throws BadRequestException {
+    try {
+      return queryProjectsProvider
+          .get()
+          .withQuery(query)
+          .withStart(start)
+          .withLimit(limit)
+          .apply()
+          .stream()
+          .collect(
+              ImmutableSortedMap.toImmutableSortedMap(
+                  natural(), p -> p.name, p -> showDescription ? p : nullifyDescription(p)));
+    } catch (OrmException | MethodNotAllowedException e) {
+      logger.atWarning().withCause(e).log(
+          "Internal error while processing the query '{}' request", query);
+      throw new BadRequestException("Internal error while processing the query request");
+    }
+  }
+
+  private ProjectInfo nullifyDescription(ProjectInfo p) {
+    p.description = null;
+    return p;
+  }
+
   public SortedMap<String, ProjectInfo> display(@Nullable OutputStream displayOutputStream)
       throws BadRequestException, PermissionBackendException {
     if (all && state != null) {
@@ -393,7 +456,7 @@ public class ListProjects implements RestReadView<TopLevelResource> {
         }
 
         if (showDescription) {
-          info.description = Strings.emptyToNull(e.getProject().getDescription());
+          info.description = emptyToNull(e.getProject().getDescription());
         }
         info.state = e.getProject().getState();