opendev
/
gerrit
Code Issues Proposed changes

Merge branch 'stable-2.14' into stable-2.15 

* stable-2.14:
  ElasticContainer: Allow to specify the docker container version to create
  ElasticContainer: Include cause in AssumptionViolatedException
  ElasticContainer: Create with static method
  Acceptance tests: Replace embedded ES with docker testcontainer
  Elasticsearch: replace native API in prod w/ REST

Changes to ReindexIT done in Iccf443102 ("Acceptance tests: Replace embedded
ES with docker testcontainer") are reverted in this merge since that test
currently doesn't work with Elasticsearch on stable-2.15 (issue 8799).

Change-Id: I21d8b10ebd450c7dc840846a5a0d836b4243dacc
This commit is contained in:
David Pursehouse 2018-05-28 19:24:39 +09:00
parent e65498896f c337290165
commit d92ba3bc8d
37 changed files with 1451 additions and 610 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

139
WORKSPACE
View File

@ -244,6 +244,12 @@ maven_jar(
sha1 = "6cca9a3b999ff28b7a35ca762b3197cd7e4c2ad1",
)
maven_jar(
name = "log_ext",
artifact = "org.slf4j:slf4j-ext:" + SLF4J_VERS,
sha1 = "09a8f58c784c37525d2624062414358acf296717",
)
maven_jar(
name = "impl_log4j",
artifact = "org.slf4j:slf4j-log4j12:" + SLF4J_VERS,
@ -477,42 +483,6 @@ maven_jar(
sha1 = "8a06fad4675473d98d93b61fea529e3f464bf69e",
)
maven_jar(
name = "lucene_highlighter",
artifact = "org.apache.lucene:lucene-highlighter:" + LUCENE_VERS,
sha1 = "433f53f03f1b14337c08d54e507a5410905376fa",
)
maven_jar(
name = "lucene_join",
artifact = "org.apache.lucene:lucene-join:" + LUCENE_VERS,
sha1 = "23f9a909a244ed3b28b37c5bb21a6e33e6c0a339",
)
maven_jar(
name = "lucene_memory",
artifact = "org.apache.lucene:lucene-memory:" + LUCENE_VERS,
sha1 = "4dbdc2e1a24837722294762a9edb479f79092ab9",
)
maven_jar(
name = "lucene_spatial",
artifact = "org.apache.lucene:lucene-spatial:" + LUCENE_VERS,
sha1 = "0217d302dc0ef4d9b8b475ffe327d83c1e0ceba5",
)
maven_jar(
name = "lucene_suggest",
artifact = "org.apache.lucene:lucene-suggest:" + LUCENE_VERS,
sha1 = "0f46dbb3229eed62dff10d008172c885e0e028c8",
)
maven_jar(
name = "lucene_queries",
artifact = "org.apache.lucene:lucene-queries:" + LUCENE_VERS,
sha1 = "f915357b8b4b43742ab48f1401dedcaa12dfa37a",
)
maven_jar(
name = "mime_util",
artifact = "eu.medsea.mimeutil:mime-util:2.1.3",
@ -680,6 +650,12 @@ maven_jar(
# Test-only dependencies below.
maven_jar(
name = "elasticsearch",
artifact = "org.elasticsearch:elasticsearch:2.4.4",
sha1 = "e69930bc794c539d34778e665d6f8ccbffd42c6f",
)
maven_jar(
name = "jimfs",
artifact = "com.google.jimfs:jimfs:1.1",
@ -909,74 +885,17 @@ maven_jar(
# When upgrading Elasticsearch, make sure it's compatible with Lucene
maven_jar(
name = "elasticsearch",
artifact = "org.elasticsearch:elasticsearch:2.4.5",
sha1 = "daafe48ae06592029a2fedca1fe2ac0f5eec3185",
name = "elasticsearch-rest-client",
artifact = "org.elasticsearch.client:elasticsearch-rest-client:5.6.9",
sha1 = "895706412e2fba3f842fca82ec3dece1cb4ee7d1",
)
# Java REST client for Elasticsearch.
JEST_VERSION = "2.4.0"
maven_jar(
name = "jest_common",
artifact = "io.searchbox:jest-common:" + JEST_VERSION,
sha1 = "ea779ebe7c438a53dce431f85b0d4e1d8faee2ac",
)
maven_jar(
name = "jest",
artifact = "io.searchbox:jest:" + JEST_VERSION,
sha1 = "e2a604a584e6633545ac6b1fe99ef888ab96dae9",
)
maven_jar(
name = "compress_lzf",
artifact = "com.ning:compress-lzf:1.0.2",
sha1 = "62896e6fca184c79cc01a14d143f3ae2b4f4b4ae",
)
maven_jar(
name = "hppc",
artifact = "com.carrotsearch:hppc:0.7.1",
sha1 = "8b5057f74ea378c0150a1860874a3ebdcb713767",
)
maven_jar(
name = "jsr166e",
artifact = "com.twitter:jsr166e:1.1.0",
sha1 = "233098147123ee5ddcd39ffc57ff648be4b7e5b2",
)
maven_jar(
name = "netty",
artifact = "io.netty:netty:3.10.0.Final",
sha1 = "ad61cd1bba067e6634ddd3e160edf0727391ac30",
)
maven_jar(
name = "t_digest",
artifact = "com.tdunning:t-digest:3.0",
sha1 = "84ccf145ac2215e6bfa63baa3101c0af41017cfc",
)
JACKSON_VERSION = "2.8.9"
JACKSON_VERSION = "2.6.6"
maven_jar(
name = "jackson_core",
artifact = "com.fasterxml.jackson.core:jackson-core:" + JACKSON_VERSION,
sha1 = "569b1752705da98f49aabe2911cc956ff7d8ed9d",
)
maven_jar(
name = "jackson_dataformat_cbor",
artifact = "com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:" + JACKSON_VERSION,
sha1 = "93242092324cad33d777e06c0515e40a6b862659",
)
maven_jar(
name = "jackson_dataformat_smile",
artifact = "com.fasterxml.jackson.dataformat:jackson-dataformat-smile:" + JACKSON_VERSION,
sha1 = "d36cbae6b06ac12fca16fda403759e479316141b",
sha1 = "02eb801df67aacaf5b1deb4ac626e1964508e47b",
)
maven_jar(
@ -991,6 +910,30 @@ maven_jar(
sha1 = "a8c5e3c3bfea5ce23fb647c335897e415eb442e3",
)
maven_jar(
name = "testcontainers",
artifact = "org.testcontainers:testcontainers:1.7.2",
sha1 = "fec8b360b6b613f6c9d3b8e7a9fa32d1a2bcb978",
)
maven_jar(
name = "duct_tape",
artifact = "org.rnorth.duct-tape:duct-tape:1.0.7",
sha1 = "a26b5d90d88c91321dc7a3734ea72d2fc019ebb6",
)
maven_jar(
name = "visible_assertions",
artifact = "org.rnorth.visible-assertions:visible-assertions:2.1.0",
sha1 = "f2fcff2862860828ac38a5e1f14d941787c06b13",
)
maven_jar(
name = "jna",
artifact = "net.java.dev.jna:jna:4.5.1",
sha1 = "65bd0cacc9c79a21c6ed8e9f588577cd3c2f85b9",
)
load("//tools/bzl:js.bzl", "npm_binary", "bower_archive")
npm_binary(

21
gerrit-elasticsearch/BUILD
View File

@ -12,14 +12,16 @@ java_library(
"//lib:guava",
"//lib:gwtorm",
"//lib/commons:codec",
"//lib/elasticsearch",
"//lib/elasticsearch-rest-client",
"//lib/guice",
"//lib/guice:guice-assistedinject",
"//lib/jest",
"//lib/jest:jest-common",
"//lib/httpcomponents:httpasyncclient",
"//lib/httpcomponents:httpclient",
"//lib/httpcomponents:httpcore",
"//lib/httpcomponents:httpcore-nio",
"//lib/jackson:jackson-core",
"//lib/jgit/org.eclipse.jgit:jgit",
"//lib/log:api",
"//lib/lucene:lucene-core",
],
)
@ -28,7 +30,11 @@ load("//tools/bzl:junit.bzl", "junit_tests")
java_library(
name = "elasticsearch_test_utils",
testonly = 1,
srcs = glob(["src/test/java/**/ElasticTestUtils.java"]),
srcs = glob([
"src/test/java/**/ElasticTestUtils.java",
"src/test/java/**/ElasticContainer.java",
]),
visibility = ["//tools/eclipse:__pkg__"],
deps = [
":elasticsearch",
"//gerrit-index:index",
@ -36,9 +42,10 @@ java_library(
"//gerrit-server:server",
"//lib:gson",
"//lib:truth",
"//lib/elasticsearch",
"//lib/guice",
"//lib/httpcomponents:httpcore",
"//lib/jgit/org.eclipse.jgit:jgit",
"//lib/testcontainers",
],
)
@ -56,7 +63,9 @@ junit_tests(
"//gerrit-server:server",
"//gerrit-server:testutil",
"//lib/guice",
"//lib/httpcomponents:httpcore",
"//lib/jgit/org.eclipse.jgit:jgit",
"//lib/jgit/org.eclipse.jgit.junit:junit",
"//lib/testcontainers",
],
)

164
gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/AbstractElasticIndex.java
View File

@ -15,14 +15,17 @@
package com.google.gerrit.elasticsearch;
import static com.google.gson.FieldNamingPolicy.LOWER_CASE_WITH_UNDERSCORES;
import static java.nio.charset.StandardCharsets.UTF_8;
import static java.util.stream.Collectors.toList;
import static org.apache.commons.codec.binary.Base64.decodeBase64;
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
import com.google.common.base.Strings;
import com.google.common.collect.FluentIterable;
import com.google.common.collect.Iterables;
import com.google.common.collect.Streams;
import com.google.common.io.CharStreams;
import com.google.gerrit.elasticsearch.builders.SearchSourceBuilder;
import com.google.gerrit.elasticsearch.builders.XContentBuilder;
import com.google.gerrit.index.Index;
import com.google.gerrit.index.Schema;
import com.google.gerrit.index.Schema.Values;
@ -33,20 +36,39 @@ import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.google.gson.JsonArray;
import com.google.gson.JsonObject;
import com.google.gson.JsonParser;
import com.google.gwtorm.protobuf.ProtobufCodec;
import io.searchbox.client.JestResult;
import io.searchbox.client.http.JestHttpClient;
import io.searchbox.core.Bulk;
import io.searchbox.core.Delete;
import io.searchbox.indices.CreateIndex;
import io.searchbox.indices.DeleteIndex;
import io.searchbox.indices.IndicesExists;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.Reader;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.http.HttpEntity;
import org.apache.http.HttpStatus;
import org.apache.http.client.methods.HttpDelete;
import org.apache.http.client.methods.HttpHead;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.methods.HttpPut;
import org.apache.http.entity.ContentType;
import org.apache.http.nio.entity.NStringEntity;
import org.eclipse.jgit.lib.Config;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.client.Response;
import org.elasticsearch.client.RestClient;
abstract class AbstractElasticIndex<K, V> implements Index<K, V> {
protected static final String BULK = "_bulk";
protected static final String DELETE = "delete";
protected static final String IGNORE_UNMAPPED = "ignore_unmapped";
protected static final String INDEX = "index";
protected static final String ORDER = "order";
protected static final String SEARCH = "_search";
protected static <T> List<T> decodeProtos(
JsonObject doc, String fieldName, ProtobufCodec<T> codec) {
JsonArray field = doc.getAsJsonArray(fieldName);
@ -58,12 +80,24 @@ abstract class AbstractElasticIndex<K, V> implements Index<K, V> {
.toList();
}
static String getContent(Response response) throws IOException {
HttpEntity responseEntity = response.getEntity();
String content = "";
if (responseEntity != null) {
InputStream contentStream = responseEntity.getContent();
try (Reader reader = new InputStreamReader(contentStream)) {
content = CharStreams.toString(reader);
}
}
return content;
}
private final Schema<V> schema;
private final SitePaths sitePaths;
private final String indexNameRaw;
private final RestClient client;
protected final String indexName;
protected final JestHttpClient client;
protected final Gson gson;
protected final ElasticQueryBuilder queryBuilder;
@ -71,7 +105,7 @@ abstract class AbstractElasticIndex<K, V> implements Index<K, V> {
@GerritServerConfig Config cfg,
SitePaths sitePaths,
Schema<V> schema,
JestClientBuilder clientBuilder,
ElasticRestClientBuilder clientBuilder,
String indexName) {
this.sitePaths = sitePaths;
this.schema = schema;
@ -94,7 +128,11 @@ abstract class AbstractElasticIndex<K, V> implements Index<K, V> {
@Override
public void close() {
client.shutdownClient();
try {
client.close();
} catch (IOException e) {
// Ignored.
}
}
@Override
@ -104,60 +142,57 @@ abstract class AbstractElasticIndex<K, V> implements Index<K, V> {
@Override
public void delete(K c) throws IOException {
Bulk bulk = addActions(new Bulk.Builder(), c).refresh(true).build();
JestResult result = client.execute(bulk);
if (!result.isSucceeded()) {
String uri = getURI(indexNameRaw, BULK);
Response response = performRequest(HttpPost.METHOD_NAME, addActions(c), uri, getRefreshParam());
int statusCode = response.getStatusLine().getStatusCode();
if (statusCode != HttpStatus.SC_OK) {
throw new IOException(
String.format(
"Failed to delete change %s in index %s: %s",
c, indexName, result.getErrorMessage()));
String.format("Failed to delete change %s in index %s: %s", c, indexName, statusCode));
}
}
@Override
public void deleteAll() throws IOException {
// Delete the index, if it exists.
JestResult result = client.execute(new IndicesExists.Builder(indexName).build());
if (result.isSucceeded()) {
result = client.execute(new DeleteIndex.Builder(indexName).build());
if (!result.isSucceeded()) {
Response response = client.performRequest(HttpHead.METHOD_NAME, indexName);
int statusCode = response.getStatusLine().getStatusCode();
if (statusCode == HttpStatus.SC_OK) {
response = client.performRequest(HttpDelete.METHOD_NAME, indexName);
statusCode = response.getStatusLine().getStatusCode();
if (statusCode != HttpStatus.SC_OK) {
throw new IOException(
String.format("Failed to delete index %s: %s", indexName, result.getErrorMessage()));
String.format("Failed to delete index %s: %s", indexName, statusCode));
}
}
// Recreate the index.
result = client.execute(new CreateIndex.Builder(indexName).settings(getMappings()).build());
if (!result.isSucceeded()) {
String error =
String.format("Failed to create index %s: %s", indexName, result.getErrorMessage());
response =
performRequest(HttpPut.METHOD_NAME, getMappings(), indexName, Collections.emptyMap());
statusCode = response.getStatusLine().getStatusCode();
if (statusCode != HttpStatus.SC_OK) {
String error = String.format("Failed to create index %s: %s", indexName, statusCode);
throw new IOException(error);
}
}
protected abstract Bulk.Builder addActions(Bulk.Builder builder, K c);
protected abstract String addActions(K c);
protected abstract String getMappings();
protected abstract String getId(V v);
protected Delete delete(String type, K c) {
protected String delete(String type, K c) {
String id = c.toString();
return new Delete.Builder(id).index(indexName).type(type).build();
}
protected io.searchbox.core.Index insert(String type, V v) throws IOException {
String id = getId(v);
String doc = toDoc(v);
return new io.searchbox.core.Index.Builder(doc).index(indexName).type(type).id(id).build();
return toAction(type, id, DELETE);
}
private static boolean shouldAddElement(Object element) {
return !(element instanceof String) || !((String) element).isEmpty();
}
private String toDoc(V v) throws IOException {
try (XContentBuilder builder = jsonBuilder().startObject()) {
protected String toDoc(V v) throws IOException {
try (XContentBuilder closeable = new XContentBuilder()) {
XContentBuilder builder = closeable.startObject();
for (Values<V> values : schema.buildFields(v)) {
String name = values.getField().getName();
if (values.getField().isRepeatable()) {
@ -173,7 +208,58 @@ abstract class AbstractElasticIndex<K, V> implements Index<K, V> {
}
}
}
return builder.endObject().string();
return builder.endObject().string() + System.lineSeparator();
}
}
protected String toAction(String type, String id, String action) {
JsonObject properties = new JsonObject();
properties.addProperty("_id", id);
properties.addProperty("_index", indexName);
properties.addProperty("_type", type);
JsonObject jsonAction = new JsonObject();
jsonAction.add(action, properties);
return jsonAction.toString() + System.lineSeparator();
}
protected void addNamedElement(String name, JsonObject element, JsonArray array) {
JsonObject arrayElement = new JsonObject();
arrayElement.add(name, element);
array.add(arrayElement);
}
protected Map<String, String> getRefreshParam() {
Map<String, String> params = new HashMap<>();
params.put("refresh", "true");
return params;
}
protected String getSearch(SearchSourceBuilder searchSource, JsonArray sortArray) {
JsonObject search = new JsonParser().parse(searchSource.toString()).getAsJsonObject();
search.add("sort", sortArray);
return gson.toJson(search);
}
protected JsonArray getSortArray(String idFieldName) {
JsonObject properties = new JsonObject();
properties.addProperty(ORDER, "asc");
properties.addProperty(IGNORE_UNMAPPED, true);
JsonArray sortArray = new JsonArray();
addNamedElement(idFieldName, properties, sortArray);
return sortArray;
}
protected String getURI(String type, String request) throws UnsupportedEncodingException {
String encodedType = URLEncoder.encode(type, UTF_8.toString());
String encodedIndexName = URLEncoder.encode(indexName, UTF_8.toString());
return encodedIndexName + "/" + encodedType + "/" + request;
}
protected Response performRequest(
String method, String payload, String uri, Map<String, String> params) throws IOException {
HttpEntity entity = new NStringEntity(payload, ContentType.APPLICATION_JSON);
return client.performRequest(method, uri, params, entity);
}
}

81
gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/ElasticAccountIndex.java
View File

@ -16,10 +16,11 @@ package com.google.gerrit.elasticsearch;
import static com.google.gerrit.server.index.account.AccountField.ID;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
import com.google.gerrit.elasticsearch.ElasticMapping.MappingProperties;
import com.google.gerrit.elasticsearch.builders.QueryBuilder;
import com.google.gerrit.elasticsearch.builders.SearchSourceBuilder;
import com.google.gerrit.index.QueryOptions;
import com.google.gerrit.index.Schema;
import com.google.gerrit.index.query.DataSource;
@ -36,51 +37,48 @@ import com.google.gerrit.server.index.account.AccountIndex;
import com.google.gson.JsonArray;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import com.google.gson.JsonParser;
import com.google.gwtorm.server.OrmException;
import com.google.gwtorm.server.ResultSet;
import com.google.inject.Inject;
import com.google.inject.Provider;
import com.google.inject.assistedinject.Assisted;
import io.searchbox.client.JestResult;
import io.searchbox.core.Bulk;
import io.searchbox.core.Bulk.Builder;
import io.searchbox.core.Search;
import io.searchbox.core.search.sort.Sort;
import io.searchbox.core.search.sort.Sort.Sorting;
import com.google.inject.assistedinject.AssistedInject;
import java.io.IOException;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
import org.apache.http.HttpStatus;
import org.apache.http.StatusLine;
import org.apache.http.client.methods.HttpPost;
import org.eclipse.jgit.lib.Config;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.client.Response;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class ElasticAccountIndex extends AbstractElasticIndex<Account.Id, AccountState>
implements AccountIndex {
static class AccountMapping {
public static class AccountMapping {
MappingProperties accounts;
AccountMapping(Schema<AccountState> schema) {
public AccountMapping(Schema<AccountState> schema) {
this.accounts = ElasticMapping.createMapping(schema);
}
}
static final String ACCOUNTS = "accounts";
public static final String ACCOUNTS = "accounts";
private static final Logger log = LoggerFactory.getLogger(ElasticAccountIndex.class);
private final AccountMapping mapping;
private final Provider<AccountCache> accountCache;
@Inject
@AssistedInject
ElasticAccountIndex(
@GerritServerConfig Config cfg,
SitePaths sitePaths,
Provider<AccountCache> accountCache,
JestClientBuilder clientBuilder,
ElasticRestClientBuilder clientBuilder,
@Assisted Schema<AccountState> schema) {
super(cfg, sitePaths, schema, clientBuilder, ACCOUNTS);
this.accountCache = accountCache;
@ -89,19 +87,17 @@ public class ElasticAccountIndex extends AbstractElasticIndex<Account.Id, Accoun
@Override
public void replace(AccountState as) throws IOException {
Bulk bulk =
new Bulk.Builder()
.defaultIndex(indexName)
.defaultType(ACCOUNTS)
.addAction(insert(ACCOUNTS, as))
.refresh(true)
.build();
JestResult result = client.execute(bulk);
if (!result.isSucceeded()) {
String bulk = toAction(ACCOUNTS, getId(as), INDEX);
bulk += toDoc(as);
String uri = getURI(ACCOUNTS, BULK);
Response response = performRequest(HttpPost.METHOD_NAME, bulk, uri, getRefreshParam());
int statusCode = response.getStatusLine().getStatusCode();
if (statusCode != HttpStatus.SC_OK) {
throw new IOException(
String.format(
"Failed to replace account %s in index %s: %s",
as.getAccount().getId(), indexName, result.getErrorMessage()));
as.getAccount().getId(), indexName, statusCode));
}
}
@ -112,8 +108,8 @@ public class ElasticAccountIndex extends AbstractElasticIndex<Account.Id, Accoun
}
@Override
protected Builder addActions(Builder builder, Account.Id c) {
return builder.addAction(delete(ACCOUNTS, c));
protected String addActions(Account.Id c) {
return delete(ACCOUNTS, c);
}
@Override
@ -128,7 +124,7 @@ public class ElasticAccountIndex extends AbstractElasticIndex<Account.Id, Accoun
}
private class QuerySource implements DataSource<AccountState> {
private final Search search;
private final String search;
private final Set<String> fields;
QuerySource(Predicate<AccountState> p, QueryOptions opts) throws QueryParseException {
@ -141,15 +137,8 @@ public class ElasticAccountIndex extends AbstractElasticIndex<Account.Id, Accoun
.size(opts.limit())
.fields(Lists.newArrayList(fields));
Sort sort = new Sort(AccountField.ID.getName(), Sorting.ASC);
sort.setIgnoreUnmapped();
search =
new Search.Builder(searchSource.toString())
.addType(ACCOUNTS)
.addIndex(indexName)
.addSort(ImmutableList.of(sort))
.build();
JsonArray sortArray = getSortArray(AccountField.ID.getName());
search = getSearch(searchSource, sortArray);
}
@Override
@ -161,9 +150,14 @@ public class ElasticAccountIndex extends AbstractElasticIndex<Account.Id, Accoun
public ResultSet<AccountState> read() throws OrmException {
try {
List<AccountState> results = Collections.emptyList();
JestResult result = client.execute(search);
if (result.isSucceeded()) {
JsonObject obj = result.getJsonObject().getAsJsonObject("hits");
String uri = getURI(ACCOUNTS, SEARCH);
Response response =
performRequest(HttpPost.METHOD_NAME, search, uri, Collections.emptyMap());
StatusLine statusLine = response.getStatusLine();
if (statusLine.getStatusCode() == HttpStatus.SC_OK) {
String content = getContent(response);
JsonObject obj =
new JsonParser().parse(content).getAsJsonObject().getAsJsonObject("hits");
if (obj.get("hits") != null) {
JsonArray json = obj.getAsJsonArray("hits");
results = Lists.newArrayListWithCapacity(json.size());
@ -172,7 +166,7 @@ public class ElasticAccountIndex extends AbstractElasticIndex<Account.Id, Accoun
}
}
} else {
log.error(result.getErrorMessage());
log.error(statusLine.getReasonPhrase());
}
final List<AccountState> r = Collections.unmodifiableList(results);
return new ResultSet<AccountState>() {
@ -196,11 +190,6 @@ public class ElasticAccountIndex extends AbstractElasticIndex<Account.Id, Accoun
}
}
@Override
public String toString() {
return search.toString();
}
private AccountState toAccountState(JsonElement json) {
JsonElement source = json.getAsJsonObject().get("_source");
if (source == null) {

96
gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/ElasticChangeIndex.java
View File

@ -24,7 +24,6 @@ import static java.nio.charset.StandardCharsets.UTF_8;
import static org.apache.commons.codec.binary.Base64.decodeBase64;
import com.google.common.collect.FluentIterable;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Iterables;
import com.google.common.collect.ListMultimap;
@ -32,6 +31,8 @@ import com.google.common.collect.Lists;
import com.google.common.collect.MultimapBuilder;
import com.google.common.collect.Sets;
import com.google.gerrit.elasticsearch.ElasticMapping.MappingProperties;
import com.google.gerrit.elasticsearch.builders.QueryBuilder;
import com.google.gerrit.elasticsearch.builders.SearchSourceBuilder;
import com.google.gerrit.index.QueryOptions;
import com.google.gerrit.index.Schema;
import com.google.gerrit.index.query.Predicate;
@ -55,26 +56,24 @@ import com.google.gerrit.server.query.change.ChangeDataSource;
import com.google.gson.JsonArray;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import com.google.gson.JsonParser;
import com.google.gwtorm.server.OrmException;
import com.google.gwtorm.server.ResultSet;
import com.google.inject.Inject;
import com.google.inject.Provider;
import com.google.inject.assistedinject.Assisted;
import io.searchbox.client.JestResult;
import io.searchbox.core.Bulk;
import io.searchbox.core.Bulk.Builder;
import io.searchbox.core.Search;
import io.searchbox.core.search.sort.Sort;
import io.searchbox.core.search.sort.Sort.Sorting;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
import org.apache.commons.codec.binary.Base64;
import org.apache.http.HttpStatus;
import org.apache.http.StatusLine;
import org.apache.http.client.methods.HttpPost;
import org.eclipse.jgit.lib.Config;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.client.Response;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -108,7 +107,7 @@ class ElasticChangeIndex extends AbstractElasticIndex<Change.Id, ChangeData>
Provider<ReviewDb> db,
ChangeData.Factory changeDataFactory,
SitePaths sitePaths,
JestClientBuilder clientBuilder,
ElasticRestClientBuilder clientBuilder,
@Assisted Schema<ChangeData> schema) {
super(cfg, sitePaths, schema, clientBuilder, CHANGES);
this.db = db;
@ -133,20 +132,17 @@ class ElasticChangeIndex extends AbstractElasticIndex<Change.Id, ChangeData>
throw new IOException(e);
}
Bulk bulk =
new Bulk.Builder()
.defaultIndex(indexName)
.defaultType(CHANGES)
.addAction(insert(insertIndex, cd))
.addAction(delete(deleteIndex, cd.getId()))
.refresh(true)
.build();
JestResult result = client.execute(bulk);
if (!result.isSucceeded()) {
String bulk = toAction(insertIndex, getId(cd), INDEX);
bulk += toDoc(cd);
bulk += toAction(deleteIndex, cd.getId().toString(), DELETE);
String uri = getURI(CHANGES, BULK);
Response response = performRequest(HttpPost.METHOD_NAME, bulk, uri, getRefreshParam());
int statusCode = response.getStatusLine().getStatusCode();
if (statusCode != HttpStatus.SC_OK) {
throw new IOException(
String.format(
"Failed to replace change %s in index %s: %s",
cd.getId(), indexName, result.getErrorMessage()));
"Failed to replace change %s in index %s: %s", cd.getId(), indexName, statusCode));
}
}
@ -165,8 +161,8 @@ class ElasticChangeIndex extends AbstractElasticIndex<Change.Id, ChangeData>
}
@Override
protected Builder addActions(Builder builder, Id c) {
return builder.addAction(delete(OPEN_CHANGES, c)).addAction(delete(OPEN_CHANGES, c));
protected String addActions(Id c) {
return delete(OPEN_CHANGES, c) + delete(CLOSED_CHANGES, c);
}
@Override
@ -180,18 +176,12 @@ class ElasticChangeIndex extends AbstractElasticIndex<Change.Id, ChangeData>
}
private class QuerySource implements ChangeDataSource {
private final Search search;
private final String search;
private final Set<String> fields;
private final List<String> types;
QuerySource(List<String> types, Predicate<ChangeData> p, QueryOptions opts)
throws QueryParseException {
List<Sort> sorts =
ImmutableList.of(
new Sort(ChangeField.UPDATED.getName(), Sorting.DESC),
new Sort(ChangeField.LEGACY_ID.getName(), Sorting.DESC));
for (Sort sort : sorts) {
sort.setIgnoreUnmapped();
}
QueryBuilder qb = queryBuilder.toQueryBuilder(p);
fields = IndexUtils.changeFields(opts);
SearchSourceBuilder searchSource =
@ -201,12 +191,8 @@ class ElasticChangeIndex extends AbstractElasticIndex<Change.Id, ChangeData>
.size(opts.limit())
.fields(Lists.newArrayList(fields));
search =
new Search.Builder(searchSource.toString())
.addType(types)
.addSort(sorts)
.addIndex(indexName)
.build();
search = getSearch(searchSource, getSortArray());
this.types = types;
}
@Override
@ -218,9 +204,14 @@ class ElasticChangeIndex extends AbstractElasticIndex<Change.Id, ChangeData>
public ResultSet<ChangeData> read() throws OrmException {
try {
List<ChangeData> results = Collections.emptyList();
JestResult result = client.execute(search);
if (result.isSucceeded()) {
JsonObject obj = result.getJsonObject().getAsJsonObject("hits");
String uri = getURI(types);
Response response =
performRequest(HttpPost.METHOD_NAME, search, uri, Collections.emptyMap());
StatusLine statusLine = response.getStatusLine();
if (statusLine.getStatusCode() == HttpStatus.SC_OK) {
String content = getContent(response);
JsonObject obj =
new JsonParser().parse(content).getAsJsonObject().getAsJsonObject("hits");
if (obj.get("hits") != null) {
JsonArray json = obj.getAsJsonArray("hits");
results = Lists.newArrayListWithCapacity(json.size());
@ -229,7 +220,7 @@ class ElasticChangeIndex extends AbstractElasticIndex<Change.Id, ChangeData>
}
}
} else {
log.error(result.getErrorMessage());
log.error(statusLine.getReasonPhrase());
}
final List<ChangeData> r = Collections.unmodifiableList(results);
return new ResultSet<ChangeData>() {
@ -258,11 +249,6 @@ class ElasticChangeIndex extends AbstractElasticIndex<Change.Id, ChangeData>
return false;
}
@Override
public String toString() {
return search.toString();
}
private ChangeData toChangeData(JsonElement json) {
JsonElement sourceElement = json.getAsJsonObject().get("_source");
if (sourceElement == null) {
@ -438,5 +424,21 @@ class ElasticChangeIndex extends AbstractElasticIndex<Change.Id, ChangeData>
}
out.setUnresolvedCommentCount(count.getAsInt());
}
private JsonArray getSortArray() {
JsonObject properties = new JsonObject();
properties.addProperty(ORDER, "desc");
properties.addProperty(IGNORE_UNMAPPED, true);
JsonArray sortArray = new JsonArray();
addNamedElement(ChangeField.UPDATED.getName(), properties, sortArray);
addNamedElement(ChangeField.LEGACY_ID.getName(), properties, sortArray);
return sortArray;
}
}
private String getURI(List<String> types) throws UnsupportedEncodingException {
String joinedTypes = String.join(",", types);
return getURI(joinedTypes, SEARCH);
}
}

30
gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/ElasticConfiguration.java
View File

@ -18,13 +18,12 @@ import com.google.common.base.MoreObjects;
import com.google.gerrit.server.config.GerritServerConfig;
import com.google.inject.Inject;
import com.google.inject.Singleton;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import org.apache.http.HttpHost;
import org.eclipse.jgit.lib.Config;
@Singleton
@ -33,7 +32,7 @@ class ElasticConfiguration {
private static final String DEFAULT_PORT = "9200";
private static final String DEFAULT_PROTOCOL = "http";
final List<String> urls;
final List<HttpHost> urls;
final String username;
final String password;
final boolean requestCompression;
@ -59,14 +58,18 @@ class ElasticConfiguration {
Set<String> subsections = cfg.getSubsections("elasticsearch");
if (subsections.isEmpty()) {
this.urls = Arrays.asList(buildUrl(DEFAULT_PROTOCOL, DEFAULT_HOST, DEFAULT_PORT));
HttpHost httpHost =
new HttpHost(DEFAULT_HOST, Integer.valueOf(DEFAULT_PORT), DEFAULT_PROTOCOL);
this.urls = Collections.singletonList(httpHost);
} else {
this.urls = new ArrayList<>(subsections.size());
for (String subsection : subsections) {
String port = getString(cfg, subsection, "port", DEFAULT_PORT);
String host = getString(cfg, subsection, "hostname", DEFAULT_HOST);
String protocol = getString(cfg, subsection, "protocol", DEFAULT_PROTOCOL);
this.urls.add(buildUrl(protocol, host, port));
HttpHost httpHost = new HttpHost(host, Integer.valueOf(port), protocol);
this.urls.add(httpHost);
}
}
}
@ -74,19 +77,4 @@ class ElasticConfiguration {
private String getString(Config cfg, String subsection, String name, String defaultValue) {
return MoreObjects.firstNonNull(cfg.getString("elasticsearch", subsection, name), defaultValue);
}
private String buildUrl(String protocol, String hostname, String port) {
try {
return new URL(protocol, hostname, Integer.parseInt(port), "").toString();
} catch (MalformedURLException | NumberFormatException e) {
throw new RuntimeException(
"Cannot build url to Elasticsearch from values: protocol="
+ protocol
+ " hostname="
+ hostname
+ " port="
+ port,
e);
}
}
}

82
gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/ElasticGroupIndex.java
View File

@ -14,10 +14,11 @@
package com.google.gerrit.elasticsearch;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
import com.google.gerrit.elasticsearch.ElasticMapping.MappingProperties;
import com.google.gerrit.elasticsearch.builders.QueryBuilder;
import com.google.gerrit.elasticsearch.builders.SearchSourceBuilder;
import com.google.gerrit.index.QueryOptions;
import com.google.gerrit.index.Schema;
import com.google.gerrit.index.query.DataSource;
@ -34,26 +35,23 @@ import com.google.gerrit.server.index.group.GroupIndex;
import com.google.gson.JsonArray;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import com.google.gson.JsonParser;
import com.google.gwtorm.server.OrmException;
import com.google.gwtorm.server.ResultSet;
import com.google.inject.Inject;
import com.google.inject.Provider;
import com.google.inject.assistedinject.Assisted;
import io.searchbox.client.JestResult;
import io.searchbox.core.Bulk;
import io.searchbox.core.Bulk.Builder;
import io.searchbox.core.Search;
import io.searchbox.core.search.sort.Sort;
import io.searchbox.core.search.sort.Sort.Sorting;
import com.google.inject.assistedinject.AssistedInject;
import java.io.IOException;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.Optional;
import java.util.Set;
import org.apache.http.HttpStatus;
import org.apache.http.StatusLine;
import org.apache.http.client.methods.HttpPost;
import org.eclipse.jgit.lib.Config;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.client.Response;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -67,19 +65,19 @@ public class ElasticGroupIndex extends AbstractElasticIndex<AccountGroup.UUID, I
}
}
static final String GROUPS = "groups";
public static final String GROUPS = "groups";
private static final Logger log = LoggerFactory.getLogger(ElasticGroupIndex.class);
private final GroupMapping mapping;
private final Provider<GroupCache> groupCache;
@Inject
@AssistedInject
ElasticGroupIndex(
@GerritServerConfig Config cfg,
SitePaths sitePaths,
Provider<GroupCache> groupCache,
JestClientBuilder clientBuilder,
ElasticRestClientBuilder clientBuilder,
@Assisted Schema<InternalGroup> schema) {
super(cfg, sitePaths, schema, clientBuilder, GROUPS);
this.groupCache = groupCache;
@ -88,19 +86,17 @@ public class ElasticGroupIndex extends AbstractElasticIndex<AccountGroup.UUID, I
@Override
public void replace(InternalGroup group) throws IOException {
Bulk bulk =
new Bulk.Builder()
.defaultIndex(indexName)
.defaultType(GROUPS)
.addAction(insert(GROUPS, group))
.refresh(true)
.build();
JestResult result = client.execute(bulk);
if (!result.isSucceeded()) {
String bulk = toAction(GROUPS, getId(group), INDEX);
bulk += toDoc(group);
String uri = getURI(GROUPS, BULK);
Response response = performRequest(HttpPost.METHOD_NAME, bulk, uri, getRefreshParam());
int statusCode = response.getStatusLine().getStatusCode();
if (statusCode != HttpStatus.SC_OK) {
throw new IOException(
String.format(
"Failed to replace group %s in index %s: %s",
group.getGroupUUID().get(), indexName, result.getErrorMessage()));
group.getGroupUUID().get(), indexName, statusCode));
}
}
@ -111,8 +107,8 @@ public class ElasticGroupIndex extends AbstractElasticIndex<AccountGroup.UUID, I
}
@Override
protected Builder addActions(Builder builder, AccountGroup.UUID c) {
return builder.addAction(delete(GROUPS, c));
protected String addActions(AccountGroup.UUID c) {
return delete(GROUPS, c);
}
@Override
@ -127,7 +123,7 @@ public class ElasticGroupIndex extends AbstractElasticIndex<AccountGroup.UUID, I
}
private class QuerySource implements DataSource<InternalGroup> {
private final Search search;
private final String search;
private final Set<String> fields;
QuerySource(Predicate<InternalGroup> p, QueryOptions opts) throws QueryParseException {
@ -140,15 +136,8 @@ public class ElasticGroupIndex extends AbstractElasticIndex<AccountGroup.UUID, I
.size(opts.limit())
.fields(Lists.newArrayList(fields));
Sort sort = new Sort(GroupField.UUID.getName(), Sorting.ASC);
sort.setIgnoreUnmapped();
search =
new Search.Builder(searchSource.toString())
.addType(GROUPS)
.addIndex(indexName)
.addSort(ImmutableList.of(sort))
.build();
JsonArray sortArray = getSortArray(GroupField.UUID.getName());
search = getSearch(searchSource, sortArray);
}
@Override
@ -160,19 +149,23 @@ public class ElasticGroupIndex extends AbstractElasticIndex<AccountGroup.UUID, I
public ResultSet<InternalGroup> read() throws OrmException {
try {
List<InternalGroup> results = Collections.emptyList();
JestResult result = client.execute(search);
if (result.isSucceeded()) {
JsonObject obj = result.getJsonObject().getAsJsonObject("hits");
String uri = getURI(GROUPS, SEARCH);
Response response =
performRequest(HttpPost.METHOD_NAME, search, uri, Collections.emptyMap());
StatusLine statusLine = response.getStatusLine();
if (statusLine.getStatusCode() == HttpStatus.SC_OK) {
String content = getContent(response);
JsonObject obj =
new JsonParser().parse(content).getAsJsonObject().getAsJsonObject("hits");
if (obj.get("hits") != null) {
JsonArray json = obj.getAsJsonArray("hits");
results = Lists.newArrayListWithCapacity(json.size());
for (int i = 0; i < json.size(); i++) {
Optional<InternalGroup> internalGroup = toInternalGroup(json.get(i));
internalGroup.ifPresent(results::add);
results.add(toAccountGroup(json.get(i)).get());
}
}
} else {
log.error(result.getErrorMessage());
log.error(statusLine.getReasonPhrase());
}
final List<InternalGroup> r = Collections.unmodifiableList(results);
return new ResultSet<InternalGroup>() {
@ -196,12 +189,7 @@ public class ElasticGroupIndex extends AbstractElasticIndex<AccountGroup.UUID, I
}
}
@Override
public String toString() {
return search.toString();
}
private Optional<InternalGroup> toInternalGroup(JsonElement json) {
private Optional<InternalGroup> toAccountGroup(JsonElement json) {
JsonElement source = json.getAsJsonObject().get("_source");
if (source == null) {
source = json.getAsJsonObject().get("fields");

22
gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/ElasticIndexVersionDiscovery.java
View File

@ -16,31 +16,37 @@ package com.google.gerrit.elasticsearch;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import com.google.gson.JsonParser;
import com.google.inject.Inject;
import com.google.inject.Singleton;
import io.searchbox.client.JestResult;
import io.searchbox.client.http.JestHttpClient;
import io.searchbox.indices.aliases.GetAliases;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map.Entry;
import org.apache.http.HttpStatus;
import org.apache.http.client.methods.HttpGet;
import org.elasticsearch.client.Response;
import org.elasticsearch.client.RestClient;
@Singleton
class ElasticIndexVersionDiscovery {
private final JestHttpClient client;
private final RestClient client;
@Inject
ElasticIndexVersionDiscovery(JestClientBuilder clientBuilder) {
ElasticIndexVersionDiscovery(ElasticRestClientBuilder clientBuilder) {
this.client = clientBuilder.build();
}
List<String> discover(String prefix, String indexName) throws IOException {
String name = prefix + indexName + "_";
JestResult result = client.execute(new GetAliases.Builder().addIndex(name + "*").build());
if (result.isSucceeded()) {
JsonObject object = result.getJsonObject().getAsJsonObject();
Response response = client.performRequest(HttpGet.METHOD_NAME, name + "*/_aliases");
int statusCode = response.getStatusLine().getStatusCode();
if (statusCode == HttpStatus.SC_OK) {
String content = AbstractElasticIndex.getContent(response);
JsonObject object = new JsonParser().parse(content).getAsJsonObject();
List<String> versions = new ArrayList<>(object.size());
for (Entry<String, JsonElement> entry : object.entrySet()) {
versions.add(entry.getKey().replace(name, ""));

31
gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/ElasticQueryBuilder.java
View File

@ -14,6 +14,9 @@
package com.google.gerrit.elasticsearch;
import com.google.gerrit.elasticsearch.builders.BoolQueryBuilder;
import com.google.gerrit.elasticsearch.builders.QueryBuilder;
import com.google.gerrit.elasticsearch.builders.QueryBuilders;
import com.google.gerrit.index.FieldDef;
import com.google.gerrit.index.FieldType;
import com.google.gerrit.index.query.AndPredicate;
@ -28,10 +31,6 @@ import com.google.gerrit.index.query.RegexPredicate;
import com.google.gerrit.index.query.TimestampRangePredicate;
import com.google.gerrit.server.query.change.AfterPredicate;
import java.time.Instant;
import org.apache.lucene.search.BooleanQuery;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
public class ElasticQueryBuilder {
@ -52,27 +51,19 @@ public class ElasticQueryBuilder {
}
private <T> BoolQueryBuilder and(Predicate<T> p) throws QueryParseException {
try {
BoolQueryBuilder b = QueryBuilders.boolQuery();
for (Predicate<T> c : p.getChildren()) {
b.must(toQueryBuilder(c));
}
return b;
} catch (BooleanQuery.TooManyClauses e) {
throw new QueryParseException("cannot create query for index: " + p, e);
BoolQueryBuilder b = QueryBuilders.boolQuery();
for (Predicate<T> c : p.getChildren()) {
b.must(toQueryBuilder(c));
}
return b;
}
private <T> BoolQueryBuilder or(Predicate<T> p) throws QueryParseException {
try {
BoolQueryBuilder q = QueryBuilders.boolQuery();
for (Predicate<T> c : p.getChildren()) {
q.should(toQueryBuilder(c));
}
return q;
} catch (BooleanQuery.TooManyClauses e) {
throw new QueryParseException("cannot create query for index: " + p, e);
BoolQueryBuilder q = QueryBuilders.boolQuery();
for (Predicate<T> c : p.getChildren()) {
q.should(toQueryBuilder(c));
}
return q;
}
private <T> QueryBuilder not(Predicate<T> p) throws QueryParseException {

58
gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/ElasticRestClientBuilder.java Normal file
View File

@ -0,0 +1,58 @@
// Copyright (C) 2018 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.elasticsearch;
import com.google.inject.Inject;
import com.google.inject.Singleton;
import org.apache.http.HttpHost;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.impl.nio.client.HttpAsyncClientBuilder;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.client.RestClientBuilder;
@Singleton
class ElasticRestClientBuilder {
private final HttpHost[] hosts;
private final String username;
private final String password;
@Inject
ElasticRestClientBuilder(ElasticConfiguration cfg) {
hosts = cfg.urls.toArray(new HttpHost[cfg.urls.size()]);
username = cfg.username;
password = cfg.password;
}
RestClient build() {
RestClientBuilder builder = RestClient.builder(hosts);
setConfiguredCredentialsIfAny(builder);
return builder.build();
}
private void setConfiguredCredentialsIfAny(RestClientBuilder builder) {
if (username != null && password != null) {
CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
credentialsProvider.setCredentials(
AuthScope.ANY, new UsernamePasswordCredentials(username, password));
builder.setHttpClientConfigCallback(
(HttpAsyncClientBuilder httpClientBuilder) ->
httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider));
}
}
}

54
gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/JestClientBuilder.java
View File

@ -1,54 +0,0 @@
// Copyright (C) 2017 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.elasticsearch;
import com.google.inject.Inject;
import com.google.inject.Singleton;
import io.searchbox.client.JestClientFactory;
import io.searchbox.client.config.HttpClientConfig;
import io.searchbox.client.config.HttpClientConfig.Builder;
import io.searchbox.client.http.JestHttpClient;
import java.util.concurrent.TimeUnit;
@Singleton
class JestClientBuilder {
private final ElasticConfiguration cfg;
@Inject
JestClientBuilder(ElasticConfiguration cfg) {
this.cfg = cfg;
}
JestHttpClient build() {
JestClientFactory factory = new JestClientFactory();
Builder builder =
new HttpClientConfig.Builder(cfg.urls)
.multiThreaded(true)
.discoveryEnabled(false)
.connTimeout((int) cfg.connectionTimeout)
.maxConnectionIdleTime(cfg.maxConnectionIdleTime, cfg.maxConnectionIdleUnit)
.maxTotalConnection(cfg.maxTotalConnection)
.readTimeout(cfg.readTimeout)
.requestCompressionEnabled(cfg.requestCompression)
.discoveryFrequency(1L, TimeUnit.MINUTES);
if (cfg.username != null && cfg.password != null) {
builder.defaultCredentials(cfg.username, cfg.password);
}
factory.setHttpClientConfig(builder.build());
return (JestHttpClient) factory.getObject();
}
}

88
gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/builders/BoolQueryBuilder.java Normal file
View File

@ -0,0 +1,88 @@
// Copyright (C) 2018 The Android Open Source Project, 2009-2015 Elasticsearch
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.elasticsearch.builders;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
/**
* A Query that matches documents matching boolean combinations of other queries. A trimmed down
* version of {@link org.elasticsearch.index.query.BoolQueryBuilder} for this very package.
*/
public class BoolQueryBuilder extends QueryBuilder {
private final List<QueryBuilder> mustClauses = new ArrayList<>();
private final List<QueryBuilder> mustNotClauses = new ArrayList<>();
private final List<QueryBuilder> filterClauses = new ArrayList<>();
private final List<QueryBuilder> shouldClauses = new ArrayList<>();
/**
* Adds a query that <b>must</b> appear in the matching documents and will contribute to scoring.
*/
public BoolQueryBuilder must(QueryBuilder queryBuilder) {
mustClauses.add(queryBuilder);
return this;
}
/**
* Adds a query that <b>must not</b> appear in the matching documents and will not contribute to
* scoring.
*/
public BoolQueryBuilder mustNot(QueryBuilder queryBuilder) {
mustNotClauses.add(queryBuilder);
return this;
}
/**
* Adds a query that <i>should</i> appear in the matching documents. For a boolean query with no
* <tt>MUST</tt> clauses one or more <code>SHOULD</code> clauses must match a document for the
* BooleanQuery to match.
*/
public BoolQueryBuilder should(QueryBuilder queryBuilder) {
shouldClauses.add(queryBuilder);
return this;
}
@Override
protected void doXContent(XContentBuilder builder) throws IOException {
builder.startObject("bool");
doXArrayContent("must", mustClauses, builder);
doXArrayContent("filter", filterClauses, builder);
doXArrayContent("must_not", mustNotClauses, builder);
doXArrayContent("should", shouldClauses, builder);
builder.endObject();
}
private void doXArrayContent(String field, List<QueryBuilder> clauses, XContentBuilder builder)
throws IOException {
if (clauses.isEmpty()) {
return;
}
if (clauses.size() == 1) {
builder.field(field);
clauses.get(0).toXContent(builder);
} else {
builder.startArray(field);
for (QueryBuilder clause : clauses) {
clause.toXContent(builder);
}
builder.endArray();
}
}
}

37
gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/builders/ExistsQueryBuilder.java Normal file
View File

@ -0,0 +1,37 @@
// Copyright (C) 2018 The Android Open Source Project, 2009-2015 Elasticsearch
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.elasticsearch.builders;
import java.io.IOException;
/**
* Constructs a query that only match on documents that the field has a value in them. A trimmed
* down version of {@link org.elasticsearch.index.query.ExistsQueryBuilder} for this very package.
*/
class ExistsQueryBuilder extends QueryBuilder {
private final String name;
ExistsQueryBuilder(String name) {
this.name = name;
}
@Override
protected void doXContent(XContentBuilder builder) throws IOException {
builder.startObject("exists");
builder.field("field", name);
builder.endObject();
}
}

30
gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/builders/MatchAllQueryBuilder.java Normal file
View File

@ -0,0 +1,30 @@
// Copyright (C) 2018 The Android Open Source Project, 2009-2015 Elasticsearch
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.elasticsearch.builders;
import java.io.IOException;
/**
* A query that matches on all documents. A trimmed down version of {@link
* org.elasticsearch.index.query.MatchAllQueryBuilder} for this very package.
*/
class MatchAllQueryBuilder extends QueryBuilder {
@Override
protected void doXContent(XContentBuilder builder) throws IOException {
builder.startObject("match_all");
builder.endObject();
}
}

64
gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/builders/MatchQueryBuilder.java Normal file
View File

@ -0,0 +1,64 @@
// Copyright (C) 2018 The Android Open Source Project, 2009-2015 Elasticsearch
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.elasticsearch.builders;
import java.io.IOException;
import java.util.Locale;
/**
* Match query is a query that analyzes the text and constructs a query as the result of the
* analysis. It can construct different queries based on the type provided. A trimmed down version
* of {@link org.elasticsearch.index.query.MatchQueryBuilder} for this very package.
*/
class MatchQueryBuilder extends QueryBuilder {
enum Type {
/** The text is analyzed and used as a phrase query. */
PHRASE,
/** The text is analyzed and used in a phrase query, with the last term acting as a prefix. */
PHRASE_PREFIX
}
private final String name;
private final Object text;
private Type type;
/** Constructs a new text query. */
MatchQueryBuilder(String name, Object text) {
this.name = name;
this.text = text;
}
/** Sets the type of the text query. */
MatchQueryBuilder type(Type type) {
this.type = type;
return this;
}
@Override
protected void doXContent(XContentBuilder builder) throws IOException {
builder.startObject("match");
builder.startObject(name);
builder.field("query", text);
if (type != null) {
builder.field("type", type.toString().toLowerCase(Locale.ENGLISH));
}
builder.endObject();
builder.endObject();
}
}

34
gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/builders/QueryBuilder.java Normal file
View File

@ -0,0 +1,34 @@
// Copyright (C) 2018 The Android Open Source Project, 2009-2015 Elasticsearch
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.elasticsearch.builders;
import java.io.IOException;
/**
* A trimmed down version of {@link org.elasticsearch.index.query.QueryBuilder} for this very
* package.
*/
public abstract class QueryBuilder {
protected QueryBuilder() {}
protected void toXContent(XContentBuilder builder) throws IOException {
builder.startObject();
doXContent(builder);
builder.endObject();
}
protected abstract void doXContent(XContentBuilder builder) throws IOException;
}

102
gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/builders/QueryBuilders.java Normal file
View File

@ -0,0 +1,102 @@
// Copyright (C) 2018 The Android Open Source Project, 2009-2015 Elasticsearch
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.elasticsearch.builders;
/**
* A static factory for simple "import static" usage. A trimmed down version of {@link
* org.elasticsearch.index.query.QueryBuilders} for this very package.
*/
public abstract class QueryBuilders {
/** A query that match on all documents. */
public static MatchAllQueryBuilder matchAllQuery() {
return new MatchAllQueryBuilder();
}
/**
* Creates a text query with type "PHRASE" for the provided field name and text.
*
* @param name The field name.
* @param text The query text (to be analyzed).
*/
public static MatchQueryBuilder matchPhraseQuery(String name, Object text) {
return new MatchQueryBuilder(name, text).type(MatchQueryBuilder.Type.PHRASE);
}
/**
* Creates a match query with type "PHRASE_PREFIX" for the provided field name and text.
*
* @param name The field name.
* @param text The query text (to be analyzed).
*/
public static MatchQueryBuilder matchPhrasePrefixQuery(String name, Object text) {
return new MatchQueryBuilder(name, text).type(MatchQueryBuilder.Type.PHRASE_PREFIX);
}
/**
* A Query that matches documents containing a term.
*
* @param name The name of the field
* @param value The value of the term
*/
public static TermQueryBuilder termQuery(String name, String value) {
return new TermQueryBuilder(name, value);
}
/**
* A Query that matches documents containing a term.
*
* @param name The name of the field
* @param value The value of the term
*/
public static TermQueryBuilder termQuery(String name, int value) {
return new TermQueryBuilder(name, value);
}
/**
* A Query that matches documents within an range of terms.
*
* @param name The field name
*/
public static RangeQueryBuilder rangeQuery(String name) {
return new RangeQueryBuilder(name);
}
/**
* A Query that matches documents containing terms with a specified regular expression.
*
* @param name The name of the field
* @param regexp The regular expression
*/
public static RegexpQueryBuilder regexpQuery(String name, String regexp) {
return new RegexpQueryBuilder(name, regexp);
}
/** A Query that matches documents matching boolean combinations of other queries. */
public static BoolQueryBuilder boolQuery() {
return new BoolQueryBuilder();
}
/**
* A filter to filter only documents where a field exists in them.
*
* @param name The name of the field
*/
public static ExistsQueryBuilder existsQuery(String name) {
return new ExistsQueryBuilder(name);
}
private QueryBuilders() {}
}

35
gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/builders/QuerySourceBuilder.java Normal file
View File

@ -0,0 +1,35 @@
// Copyright (C) 2018 The Android Open Source Project, 2009-2015 Elasticsearch
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.elasticsearch.builders;
import java.io.IOException;
/**
* A trimmed down and further altered version of {@link
* org.elasticsearch.action.support.QuerySourceBuilder} for this very package.
*/
class QuerySourceBuilder {
private final QueryBuilder queryBuilder;
QuerySourceBuilder(QueryBuilder queryBuilder) {
this.queryBuilder = queryBuilder;
}
void innerToXContent(XContentBuilder builder) throws IOException {
builder.field("query");
queryBuilder.toXContent(builder);
}
}

88
gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/builders/RangeQueryBuilder.java Normal file
View File

@ -0,0 +1,88 @@
// Copyright (C) 2018 The Android Open Source Project, 2009-2015 Elasticsearch
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.elasticsearch.builders;
import java.io.IOException;
/**
* A Query that matches documents within an range of terms. A trimmed down version of {@link
* org.elasticsearch.index.query.RangeQueryBuilder} for this very package.
*/
public class RangeQueryBuilder extends QueryBuilder {
private final String name;
private Object from;
private Object to;
private boolean includeLower = true;
private boolean includeUpper = true;
/**
* A Query that matches documents within an range of terms.
*
* @param name The field name
*/
RangeQueryBuilder(String name) {
this.name = name;
}
/** The from part of the range query. Null indicates unbounded. */
public RangeQueryBuilder gt(Object from) {
this.from = from;
this.includeLower = false;
return this;
}
/** The from part of the range query. Null indicates unbounded. */
public RangeQueryBuilder gte(Object from) {
this.from = from;
this.includeLower = true;
return this;
}
/** The from part of the range query. Null indicates unbounded. */
public RangeQueryBuilder gte(int from) {
this.from = from;
this.includeLower = true;
return this;
}
/** The to part of the range query. Null indicates unbounded. */
public RangeQueryBuilder lte(Object to) {
this.to = to;
this.includeUpper = true;
return this;
}
/** The to part of the range query. Null indicates unbounded. */
public RangeQueryBuilder lte(int to) {
this.to = to;
this.includeUpper = true;
return this;
}
@Override
protected void doXContent(XContentBuilder builder) throws IOException {
builder.startObject("range");
builder.startObject(name);
builder.field("from", from);
builder.field("to", to);
builder.field("include_lower", includeLower);
builder.field("include_upper", includeUpper);
builder.endObject();
builder.endObject();
}
}

50
gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/builders/RegexpQueryBuilder.java Normal file
View File

@ -0,0 +1,50 @@
// Copyright (C) 2018 The Android Open Source Project, 2009-2015 Elasticsearch
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.elasticsearch.builders;
import java.io.IOException;
/**
* A Query that does fuzzy matching for a specific value. A trimmed down version of {@link
* org.elasticsearch.index.query.RegexpQueryBuilder} for this very package.
*/
class RegexpQueryBuilder extends QueryBuilder {
private final String name;
private final String regexp;
/**
* Constructs a new term query.
*
* @param name The name of the field
* @param regexp The regular expression
*/
RegexpQueryBuilder(String name, String regexp) {
this.name = name;
this.regexp = regexp;
}
@Override
protected void doXContent(XContentBuilder builder) throws IOException {
builder.startObject("regexp");
builder.startObject(name);
builder.field("value", regexp);
builder.field("flags_value", 65535);
builder.endObject();
builder.endObject();
}
}

108
gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/builders/SearchSourceBuilder.java Normal file
View File

@ -0,0 +1,108 @@
// Copyright (C) 2018 The Android Open Source Project, 2009-2015 Elasticsearch
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.elasticsearch.builders;
import java.io.IOException;
import java.util.List;
/**
* A search source builder allowing to easily build search source. A trimmed down and further
* altered version of {@link org.elasticsearch.search.builder.SearchSourceBuilder} for this very
* package.
*/
public class SearchSourceBuilder {
private QuerySourceBuilder querySourceBuilder;
private int from = -1;
private int size = -1;
private List<String> fieldNames;
/** Constructs a new search source builder. */
public SearchSourceBuilder() {}
/** Constructs a new search source builder with a search query. */
public SearchSourceBuilder query(QueryBuilder query) {
if (this.querySourceBuilder == null) {
this.querySourceBuilder = new QuerySourceBuilder(query);
}
return this;
}
/** From index to start the search from. Defaults to <tt>0</tt>. */
public SearchSourceBuilder from(int from) {
this.from = from;
return this;
}
/** The number of search hits to return. Defaults to <tt>10</tt>. */
public SearchSourceBuilder size(int size) {
this.size = size;
return this;
}
/**
* Sets the fields to load and return as part of the search request. If none are specified, the
* source of the document will be returned.
*/
public SearchSourceBuilder fields(List<String> fields) {
this.fieldNames = fields;
return this;
}
@Override
public final String toString() {
try {
XContentBuilder builder = new XContentBuilder();
toXContent(builder);
return builder.string();
} catch (IOException ioe) {
return "";
}
}
private void toXContent(XContentBuilder builder) throws IOException {
builder.startObject();
innerToXContent(builder);
builder.endObject();
}
private void innerToXContent(XContentBuilder builder) throws IOException {
if (from != -1) {
builder.field("from", from);
}
if (size != -1) {
builder.field("size", size);
}
if (querySourceBuilder != null) {
querySourceBuilder.innerToXContent(builder);
}
if (fieldNames != null) {
if (fieldNames.size() == 1) {
builder.field("fields", fieldNames.get(0));
} else {
builder.startArray("fields");
for (String fieldName : fieldNames) {
builder.value(fieldName);
}
builder.endArray();
}
}
}
}

66
gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/builders/TermQueryBuilder.java Normal file
View File

@ -0,0 +1,66 @@
// Copyright (C) 2018 The Android Open Source Project, 2009-2015 Elasticsearch
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.elasticsearch.builders;
import java.io.IOException;
/**
* A Query that matches documents containing a term. A trimmed down version of {@link
* org.elasticsearch.index.query.TermQueryBuilder} for this very package.
*/
class TermQueryBuilder extends QueryBuilder {
private final String name;
private final Object value;
/**
* Constructs a new term query.
*
* @param name The name of the field
* @param value The value of the term
*/
TermQueryBuilder(String name, String value) {
this(name, (Object) value);
}
/**
* Constructs a new term query.
*
* @param name The name of the field
* @param value The value of the term
*/
TermQueryBuilder(String name, int value) {
this(name, (Object) value);
}
/**
* Constructs a new term query.
*
* @param name The name of the field
* @param value The value of the term
*/
private TermQueryBuilder(String name, Object value) {
this.name = name;
this.value = value;
}
@Override
protected void doXContent(XContentBuilder builder) throws IOException {
builder.startObject("term");
builder.field(name, value);
builder.endObject();
}
}

170
gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/builders/XContentBuilder.java Normal file
View File

@ -0,0 +1,170 @@
// Copyright (C) 2018 The Android Open Source Project, 2009-2015 Elasticsearch
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.elasticsearch.builders;
import static java.time.format.DateTimeFormatter.ISO_INSTANT;
import com.fasterxml.jackson.core.JsonEncoding;
import com.fasterxml.jackson.core.JsonFactory;
import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.core.JsonParser;
import com.google.common.base.Charsets;
import java.io.ByteArrayOutputStream;
import java.io.Closeable;
import java.io.IOException;
import java.util.Date;
/**
* A trimmed down and further altered version of {@link
* org.elasticsearch.common.xcontent.XContentBuilder} for this very package.
*/
public final class XContentBuilder implements Closeable {
private final JsonGenerator generator;
private final ByteArrayOutputStream bos = new ByteArrayOutputStream();
/**
* Constructs a new builder. Make sure to call {@link #close()} when the builder is done with.
* Inspired from {@link org.elasticsearch.common.xcontent.json.JsonXContent} static block.
*/
public XContentBuilder() throws IOException {
JsonFactory jsonFactory = new JsonFactory();
jsonFactory.configure(JsonParser.Feature.ALLOW_UNQUOTED_FIELD_NAMES, true);
jsonFactory.configure(JsonGenerator.Feature.QUOTE_FIELD_NAMES, true);
jsonFactory.configure(JsonParser.Feature.ALLOW_COMMENTS, true);
jsonFactory.configure(
JsonFactory.Feature.FAIL_ON_SYMBOL_HASH_OVERFLOW,
false); // this trips on many mappings now...
this.generator = jsonFactory.createGenerator(bos, JsonEncoding.UTF8);
}
public XContentBuilder startObject(String name) throws IOException {
field(name);
startObject();
return this;
}
public XContentBuilder startObject() throws IOException {
generator.writeStartObject();
return this;
}
public XContentBuilder endObject() throws IOException {
generator.writeEndObject();
return this;
}
public void startArray(String name) throws IOException {
field(name);
startArray();
}
private void startArray() throws IOException {
generator.writeStartArray();
}
public void endArray() throws IOException {
generator.writeEndArray();
}
public XContentBuilder field(String name) throws IOException {
generator.writeFieldName(name);
return this;
}
public XContentBuilder field(String name, String value) throws IOException {
field(name);
generator.writeString(value);
return this;
}
public XContentBuilder field(String name, int value) throws IOException {
field(name);
generator.writeNumber(value);
return this;
}
public XContentBuilder field(String name, Iterable<?> value) throws IOException {
startArray(name);
for (Object o : value) {
value(o);
}
endArray();
return this;
}
public XContentBuilder field(String name, Object value) throws IOException {
field(name);
writeValue(value);
return this;
}
public XContentBuilder value(Object value) throws IOException {
writeValue(value);
return this;
}
public XContentBuilder field(String name, boolean value) throws IOException {
field(name);
generator.writeBoolean(value);
return this;
}
public XContentBuilder value(String value) throws IOException {
generator.writeString(value);
return this;
}
@Override
public void close() {
try {
generator.close();
} catch (IOException e) {
// ignore
}
}
/** Returns a string representation of the builder (only applicable for text based xcontent). */
public String string() {
close();
byte[] bytesArray = bos.toByteArray();
return new String(bytesArray, Charsets.UTF_8);
}
private void writeValue(Object value) throws IOException {
if (value == null) {
generator.writeNull();
return;
}
Class<?> type = value.getClass();
if (type == String.class) {
generator.writeString((String) value);
} else if (type == Integer.class) {
generator.writeNumber(((Integer) value));
} else if (type == byte[].class) {
generator.writeBinary((byte[]) value);
} else if (value instanceof Date) {
generator.writeString(ISO_INSTANT.format(((Date) value).toInstant()));
} else {
// if this is a "value" object, like enum, DistanceUnit, ..., just toString it
// yea, it can be misleading when toString a Java class, but really, jackson should be used in
// that case
generator.writeString(value.toString());
// throw new ElasticsearchIllegalArgumentException("type not supported for generic value
// conversion: " + type);
}
}
}

81
gerrit-elasticsearch/src/test/java/com/google/gerrit/elasticsearch/ElasticContainer.java Normal file
View File

@ -0,0 +1,81 @@
// Copyright (C) 2018 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.elasticsearch;
import com.google.common.collect.ImmutableSet;
import java.util.Set;
import org.apache.http.HttpHost;
import org.junit.internal.AssumptionViolatedException;
import org.testcontainers.containers.GenericContainer;
/* Helper class for running ES integration tests in docker container */
public class ElasticContainer<SELF extends ElasticContainer<SELF>> extends GenericContainer<SELF> {
private static final int ELASTICSEARCH_DEFAULT_PORT = 9200;
public enum Version {
V2,
V5,
V6
}
public static ElasticContainer<?> createAndStart(Version version) {
// Assumption violation is not natively supported by Testcontainers.
// See https://github.com/testcontainers/testcontainers-java/issues/343
try {
ElasticContainer<?> container = new ElasticContainer<>(version);
container.start();
return container;
} catch (Throwable t) {
throw new AssumptionViolatedException("Unable to start container", t);
}
}
public static ElasticContainer<?> createAndStart() {
return createAndStart(Version.V2);
}
private static String getImageName(Version version) {
switch (version) {
case V2:
return "elasticsearch:2.4.6-alpine";
case V5:
return "elasticsearch:5.6.9-alpine";
case V6:
return "docker.elastic.co/elasticsearch/elasticsearch:6.2.4";
}
throw new IllegalStateException("Unsupported version: " + version.name());
}
private ElasticContainer(Version version) {
super(getImageName(version));
}
@Override
protected void configure() {
addExposedPort(ELASTICSEARCH_DEFAULT_PORT);
// https://github.com/docker-library/elasticsearch/issues/58
addEnv("-Ees.network.host", "0.0.0.0");
}
@Override
protected Set<Integer> getLivenessCheckPorts() {
return ImmutableSet.of(getMappedPort(ELASTICSEARCH_DEFAULT_PORT));
}
public HttpHost getHttpHost() {
return new HttpHost(getContainerIpAddress(), getMappedPort(ELASTICSEARCH_DEFAULT_PORT));
}
}

14
gerrit-elasticsearch/src/test/java/com/google/gerrit/elasticsearch/ElasticQueryAccountsTest.java
View File

@ -19,29 +19,29 @@ import com.google.gerrit.server.query.account.AbstractQueryAccountsTest;
import com.google.gerrit.testutil.InMemoryModule;
import com.google.inject.Guice;
import com.google.inject.Injector;
import java.util.concurrent.ExecutionException;
import org.eclipse.jgit.lib.Config;
import org.junit.AfterClass;
import org.junit.BeforeClass;
public class ElasticQueryAccountsTest extends AbstractQueryAccountsTest {
private static ElasticNodeInfo nodeInfo;
private static ElasticContainer<?> container;
@BeforeClass
public static void startIndexService() throws InterruptedException, ExecutionException {
public static void startIndexService() {
if (nodeInfo != null) {
// do not start Elasticsearch twice
return;
}
nodeInfo = ElasticTestUtils.startElasticsearchNode();
container = ElasticContainer.createAndStart();
nodeInfo = new ElasticNodeInfo(container.getHttpHost().getPort());
}
@AfterClass
public static void stopElasticsearchServer() {
if (nodeInfo != null) {
nodeInfo.node.close();
nodeInfo.elasticDir.delete();
nodeInfo = null;
if (container != null) {
container.stop();
}
}

14
gerrit-elasticsearch/src/test/java/com/google/gerrit/elasticsearch/ElasticQueryChangesTest.java
View File

@ -20,7 +20,6 @@ import com.google.gerrit.testutil.InMemoryModule;
import com.google.gerrit.testutil.InMemoryRepositoryManager.Repo;
import com.google.inject.Guice;
import com.google.inject.Injector;
import java.util.concurrent.ExecutionException;
import org.eclipse.jgit.junit.TestRepository;
import org.eclipse.jgit.lib.Config;
import org.junit.AfterClass;
@ -29,22 +28,23 @@ import org.junit.Test;
public class ElasticQueryChangesTest extends AbstractQueryChangesTest {
private static ElasticNodeInfo nodeInfo;
private static ElasticContainer<?> container;
@BeforeClass
public static void startIndexService() throws InterruptedException, ExecutionException {
public static void startIndexService() {
if (nodeInfo != null) {
// do not start Elasticsearch twice
return;
}
nodeInfo = ElasticTestUtils.startElasticsearchNode();
container = ElasticContainer.createAndStart();
nodeInfo = new ElasticNodeInfo(container.getHttpHost().getPort());
}
@AfterClass
public static void stopElasticsearchServer() {
if (nodeInfo != null) {
nodeInfo.node.close();
nodeInfo.elasticDir.delete();
nodeInfo = null;
if (container != null) {
container.stop();
}
}

14
gerrit-elasticsearch/src/test/java/com/google/gerrit/elasticsearch/ElasticQueryGroupsTest.java
View File

@ -19,29 +19,29 @@ import com.google.gerrit.server.query.group.AbstractQueryGroupsTest;
import com.google.gerrit.testutil.InMemoryModule;
import com.google.inject.Guice;
import com.google.inject.Injector;
import java.util.concurrent.ExecutionException;
import org.eclipse.jgit.lib.Config;
import org.junit.AfterClass;
import org.junit.BeforeClass;
public class ElasticQueryGroupsTest extends AbstractQueryGroupsTest {
private static ElasticNodeInfo nodeInfo;
private static ElasticContainer<?> container;
@BeforeClass
public static void startIndexService() throws InterruptedException, ExecutionException {
public static void startIndexService() {
if (nodeInfo != null) {
// do not start Elasticsearch twice
return;
}
nodeInfo = ElasticTestUtils.startElasticsearchNode();
container = ElasticContainer.createAndStart();
nodeInfo = new ElasticNodeInfo(container.getHttpHost().getPort());
}
@AfterClass
public static void stopElasticsearchServer() {
if (nodeInfo != null) {
nodeInfo.node.close();
nodeInfo.elasticDir.delete();
nodeInfo = null;
if (container != null) {
container.stop();
}
}

91
gerrit-elasticsearch/src/test/java/com/google/gerrit/elasticsearch/ElasticTestUtils.java
View File

@ -14,90 +14,31 @@
package com.google.gerrit.elasticsearch;
import static com.google.common.truth.Truth.assertThat;
import com.google.common.base.Strings;
import com.google.common.io.Files;
import com.google.gerrit.index.IndexDefinition;
import com.google.gerrit.server.index.IndexModule.IndexType;
import com.google.gson.FieldNamingPolicy;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.google.inject.Injector;
import com.google.inject.Key;
import com.google.inject.TypeLiteral;
import java.io.File;
import java.io.IOException;
import java.nio.file.Path;
import java.util.Collection;
import java.util.Iterator;
import java.util.Map;
import java.util.concurrent.ExecutionException;
import org.eclipse.jgit.lib.Config;
import org.elasticsearch.action.admin.cluster.node.info.NodesInfoRequest;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.node.Node;
import org.elasticsearch.node.NodeBuilder;
public final class ElasticTestUtils {
public static class ElasticNodeInfo {
public final Node node;
public final String port;
public final File elasticDir;
public final int port;
private ElasticNodeInfo(Node node, File rootDir, String port) {
this.node = node;
public ElasticNodeInfo(int port) {
this.port = port;
this.elasticDir = rootDir;
}
}
static void configure(Config config, String port, String prefix) {
public static void configure(Config config, int port, String prefix) {
config.setEnum("index", null, "type", IndexType.ELASTICSEARCH);
config.setString("elasticsearch", "test", "protocol", "http");
config.setString("elasticsearch", "test", "hostname", "localhost");
config.setString("elasticsearch", "test", "port", port);
config.setInt("elasticsearch", "test", "port", port);
config.setString("elasticsearch", null, "prefix", prefix);
}
static ElasticNodeInfo startElasticsearchNode() throws InterruptedException, ExecutionException {
File elasticDir = Files.createTempDir();
Path elasticDirPath = elasticDir.toPath();
Settings settings =
Settings.settingsBuilder()
.put("cluster.name", "gerrit")
.put("node.name", "Gerrit Elasticsearch Test Node")
.put("node.local", true)
.put("discovery.zen.ping.multicast.enabled", false)
.put("index.store.fs.memory.enabled", true)
.put("index.gateway.type", "none")
.put("index.max_result_window", Integer.MAX_VALUE)
.put("gateway.type", "default")
.put("http.port", 0)
.put("discovery.zen.ping.unicast.hosts", "[\"localhost\"]")
.put("path.home", elasticDirPath.toAbsolutePath())
.put("path.data", elasticDirPath.resolve("data").toAbsolutePath())
.put("path.work", elasticDirPath.resolve("work").toAbsolutePath())
.put("path.logs", elasticDirPath.resolve("logs").toAbsolutePath())
.put("transport.tcp.connect_timeout", "60s")
.build();
// Start the node
Node node = NodeBuilder.nodeBuilder().settings(settings).node();
// Wait for it to be ready
node.client().admin().cluster().prepareHealth().setWaitForYellowStatus().execute().actionGet();
assertThat(node.isClosed()).isFalse();
return new ElasticNodeInfo(node, elasticDir, getHttpPort(node));
}
static class NodeInfo {
String httpAddress;
}
static class Info {
Map<String, NodeInfo> nodes;
config.setInt("index", null, "maxLimit", 10000);
}
public static void createAllIndexes(Injector injector) throws IOException {
@ -108,28 +49,6 @@ public final class ElasticTestUtils {
}
}
private static String getHttpPort(Node node) throws InterruptedException, ExecutionException {
String nodes =
node.client().admin().cluster().nodesInfo(new NodesInfoRequest("*")).get().toString();
Gson gson =
new GsonBuilder()
.setFieldNamingPolicy(FieldNamingPolicy.LOWER_CASE_WITH_UNDERSCORES)
.create();
Info info = gson.fromJson(nodes, Info.class);
if (info.nodes == null || info.nodes.size() != 1) {
throw new RuntimeException("Cannot extract local Elasticsearch http port");
}
Iterator<NodeInfo> values = info.nodes.values().iterator();
String httpAddress = values.next().httpAddress;
if (Strings.isNullOrEmpty(httpAddress)) {
throw new RuntimeException("Cannot extract local Elasticsearch http port");
}
if (httpAddress.indexOf(':') < 0) {
throw new RuntimeException("Seems that port is not included in Elasticsearch http_address");
}
return httpAddress.substring(httpAddress.indexOf(':') + 1, httpAddress.length());
}
private ElasticTestUtils() {
// hide default constructor
}

5
lib/LICENSE-elasticsearch Normal file
View File

@ -0,0 +1,5 @@
Elasticsearch
Copyright 2009-2015 Elasticsearch
This product includes software developed by The Apache Software
Foundation (http://www.apache.org/).

22
lib/LICENSE-testcontainers Normal file
View File

@ -0,0 +1,22 @@
The MIT License (MIT)
Copyright (c) 2015 Richard North
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

8
lib/elasticsearch-rest-client/BUILD Normal file
View File

@ -0,0 +1,8 @@
package(default_visibility = ["//visibility:public"])
java_library(
name = "elasticsearch-rest-client",
data = ["//lib:LICENSE-elasticsearch"],
visibility = ["//visibility:public"],
exports = ["@elasticsearch-rest-client//jar"],
)

58
lib/elasticsearch/BUILD
View File

@ -1,58 +0,0 @@
package(default_visibility = ["//visibility:public"])
java_library(
name = "elasticsearch",
data = ["//lib:LICENSE-Apache2.0"],
exports = ["@elasticsearch//jar"],
runtime_deps = [
":compress-lzf",
":hppc",
":jsr166e",
":netty",
":t-digest",
"//lib/jackson:jackson-core",
"//lib/jackson:jackson-dataformat-cbor",
"//lib/jackson:jackson-dataformat-smile",
"//lib/lucene:lucene-highlighter",
"//lib/lucene:lucene-join",
"//lib/lucene:lucene-memory",
"//lib/lucene:lucene-queries",
"//lib/lucene:lucene-spatial",
"//lib/lucene:lucene-suggest",
],
)
java_library(
name = "compress-lzf",
data = ["//lib:LICENSE-Apache2.0"],
visibility = ["//lib/elasticsearch:__pkg__"],
exports = ["@compress_lzf//jar"],
)
java_library(
name = "hppc",
data = ["//lib:LICENSE-Apache2.0"],
visibility = ["//lib/elasticsearch:__pkg__"],
exports = ["@hppc//jar"],
)
java_library(
name = "jsr166e",
data = ["//lib:LICENSE-Apache2.0"],
visibility = ["//lib/elasticsearch:__pkg__"],
exports = ["@jsr166e//jar"],
)
java_library(
name = "netty",
data = ["//lib:LICENSE-Apache2.0"],
visibility = ["//lib/elasticsearch:__pkg__"],
exports = ["@netty//jar"],
)
java_library(
name = "t-digest",
data = ["//lib:LICENSE-Apache2.0"],
visibility = ["//lib/elasticsearch:__pkg__"],
exports = ["@t_digest//jar"],
)

23
lib/jest/BUILD
View File

@ -1,23 +0,0 @@
package(default_visibility = ["//visibility:public"])
java_library(
name = "jest-common",
data = ["//lib:LICENSE-Apache2.0"],
visibility = ["//visibility:public"],
exports = ["@jest_common//jar"],
runtime_deps = [
"//lib/commons:lang3",
],
)
java_library(
name = "jest",
data = ["//lib:LICENSE-Apache2.0"],
visibility = ["//visibility:public"],
exports = ["@jest//jar"],
runtime_deps = [
"//lib/httpcomponents:httpasyncclient",
"//lib/httpcomponents:httpclient",
"//lib/httpcomponents:httpcore-nio",
],
)

7
lib/log/BUILD
View File

@ -13,6 +13,13 @@ java_library(
runtime_deps = [":api"],
)
java_library(
name = "ext",
data = ["//lib:LICENSE-slf4j"],
visibility = ["//visibility:public"],
exports = ["@log_ext//jar"],
)
java_library(
name = "impl_log4j",
data = ["//lib:LICENSE-slf4j"],

36
lib/lucene/BUILD
View File

@ -44,39 +44,3 @@ java_library(
exports = ["@lucene_queryparser//jar"],
runtime_deps = [":lucene-core-and-backward-codecs"],
)
java_library(
name = "lucene-highlighter",
data = ["//lib:LICENSE-Apache2.0"],
exports = ["@lucene_highlighter//jar"],
)
java_library(
name = "lucene-join",
data = ["//lib:LICENSE-Apache2.0"],
exports = ["@lucene_join//jar"],
)
java_library(
name = "lucene-memory",
data = ["//lib:LICENSE-Apache2.0"],
exports = ["@lucene_memory//jar"],
)
java_library(
name = "lucene-spatial",
data = ["//lib:LICENSE-Apache2.0"],
exports = ["@lucene_spatial//jar"],
)
java_library(
name = "lucene-suggest",
data = ["//lib:LICENSE-Apache2.0"],
exports = ["@lucene_suggest//jar"],
)
java_library(
name = "lucene-queries",
data = ["//lib:LICENSE-Apache2.0"],
exports = ["@lucene_queries//jar"],
)

37
lib/testcontainers/BUILD Normal file
View File

@ -0,0 +1,37 @@
java_library(
name = "duct-tape",
testonly = 1,
data = ["//lib:LICENSE-testcontainers"],
visibility = ["//visibility:public"],
exports = ["@duct_tape//jar"],
)
java_library(
name = "visible-assertions",
testonly = 1,
data = ["//lib:LICENSE-testcontainers"],
visibility = ["//visibility:public"],
exports = ["@visible_assertions//jar"],
)
java_library(
name = "jna",
testonly = 1,
data = ["//lib:LICENSE-Apache2.0"],
visibility = ["//visibility:public"],
exports = ["@jna//jar"],
)
java_library(
name = "testcontainers",
testonly = 1,
data = ["//lib:LICENSE-testcontainers"],
visibility = ["//visibility:public"],
exports = ["@testcontainers//jar"],
runtime_deps = [
":duct-tape",
":jna",
":visible-assertions",
"//lib/log:ext",
],
)

1
tools/eclipse/BUILD
View File

@ -8,6 +8,7 @@ load(
)
TEST_DEPS = [
"//gerrit-elasticsearch:elasticsearch_test_utils",
"//gerrit-gpg:gpg_tests",
"//gerrit-gwtui:ui_tests",
"//gerrit-httpd:httpd_tests",