diff --git a/Documentation/config-gerrit.txt b/Documentation/config-gerrit.txt index d796fa406d..0a2853484f 100644 --- a/Documentation/config-gerrit.txt +++ b/Documentation/config-gerrit.txt @@ -19,6 +19,26 @@ Sample `etc/gerrit.config`: directory = /var/cache/gerrit2 ---- +[[accountPatchReviewDb]] +=== Section accountPatchReviewDb + +[[accountPatchReviewDb.url]]accountPatchReviewDb.url:: ++ +The url of accountPatchReviewDb. Supported types are `H2`, `POSTGRESQL`, and +`MYSQL`. Drop the driver jar in the lib folder of the site path if the Jdbc +driver of the corresponding Database is not yet in the class path. ++ +Default is to create H2 database in the db folder of the site path. ++ +Changing this parameter requires to migrate database using the +link:pgm-MigrateAccountPatchReviewDb.html[MigrateAccountPatchReviewDb program] +before restarting the Gerrit server. + +---- +[accountPatchReviewDb] + url = jdbc:postgresql://:/?user=&password= +---- + [[accounts]] === Section accounts diff --git a/Documentation/pgm-MigrateAccountPatchReviewDb.txt b/Documentation/pgm-MigrateAccountPatchReviewDb.txt new file mode 100644 index 0000000000..e353ffdf85 --- /dev/null +++ b/Documentation/pgm-MigrateAccountPatchReviewDb.txt @@ -0,0 +1,62 @@ += MigrateAccountPatchReviewDb + +== NAME +MigrateAccountPatchReviewDb - Migrates account patch review db from one database +backend to another. + +== SYNOPSIS +[verse] +-- +_java_ -jar gerrit.war MigrateAccountPatchReviewDb + -d + [--sourceUrl] [--chunkSize] +-- + +== DESCRIPTION +Migrates AccountPatchReviewDb from one database backend to another. The +AccountPatchReviewDb is a database used to store the user file reviewed flags. + +This task is only intended to be run if the configuration parameter +link:config-gerrit.html#accountPatchReviewDb.url[accountPatchReviewDb.url] +is set or changed. + +To migrate AccountPatchReviewDb: + +* Stop Gerrit +* Configure new value for link:config-gerrit.html#accountPatchReviewDb.url[accountPatchReviewDb.url] +* Migrate data using this task +* Start Gerrit + +== OPTIONS + +-d:: +--sourceUrl:: + Url of source database. Only need to be specified if the source is not H2. + +--chunkSize:: + Chunk size of fetching from source and pushing to target on each time. + Defaults to 100000. + +== CONTEXT +This command can only be run on a server which has direct +connectivity to the database. + +== EXAMPLES +To migrate from H2 to the database specified by +link:config-gerrit.html#accountPatchReviewDb.url[accountPatchReviewDb.url] +in gerrit.config: + +---- + $ java -jar gerrit.war MigrateAccountPatchReviewDb +---- + +== SEE ALSO + +* Configuration parameter link:config-gerrit.html#accountPatchReviewDb.url[accountPatchReviewDb.url] + +GERRIT +------ +Part of link:index.html[Gerrit Code Review] + +SEARCHBOX +--------- diff --git a/Documentation/pgm-index.txt b/Documentation/pgm-index.txt index 0c347f4571..d61cc0b8f8 100644 --- a/Documentation/pgm-index.txt +++ b/Documentation/pgm-index.txt @@ -41,6 +41,9 @@ link:pgm-passwd.html[passwd]:: link:pgm-LocalUsernamesToLowerCase.html[LocalUsernamesToLowerCase]:: Convert the local username of every account to lower case. +link:pgm-MigrateAccountPatchReviewDb.html[MigrateAccountPatchReviewDb]:: + Migrates AccountPatchReviewDb from one database backend to another. + GERRIT ------ Part of link:index.html[Gerrit Code Review] diff --git a/gerrit-pgm/src/main/java/com/google/gerrit/pgm/Daemon.java b/gerrit-pgm/src/main/java/com/google/gerrit/pgm/Daemon.java index 09fe5a3665..3af1397671 100644 --- a/gerrit-pgm/src/main/java/com/google/gerrit/pgm/Daemon.java +++ b/gerrit-pgm/src/main/java/com/google/gerrit/pgm/Daemon.java @@ -74,6 +74,7 @@ import com.google.gerrit.server.plugins.PluginGuiceEnvironment; import com.google.gerrit.server.plugins.PluginRestApiModule; import com.google.gerrit.server.schema.DataSourceProvider; import com.google.gerrit.server.schema.H2AccountPatchReviewStore; +import com.google.gerrit.server.schema.JdbcAccountPatchReviewStore; import com.google.gerrit.server.schema.SchemaVersionCheck; import com.google.gerrit.server.securestore.DefaultSecureStore; import com.google.gerrit.server.securestore.SecureStore; @@ -346,7 +347,7 @@ public class Daemon extends SiteProgram { modules.add(new EventBroker.Module()); modules.add(test ? new H2AccountPatchReviewStore.InMemoryModule() - : new H2AccountPatchReviewStore.Module()); + : new JdbcAccountPatchReviewStore.Module(config)); modules.add(new ReceiveCommitsExecutorModule()); modules.add(new DiffExecutorModule()); modules.add(new MimeUtil2Module()); diff --git a/gerrit-pgm/src/main/java/com/google/gerrit/pgm/MigrateAccountPatchReviewDb.java b/gerrit-pgm/src/main/java/com/google/gerrit/pgm/MigrateAccountPatchReviewDb.java new file mode 100644 index 0000000000..8de7bea6a4 --- /dev/null +++ b/gerrit-pgm/src/main/java/com/google/gerrit/pgm/MigrateAccountPatchReviewDb.java @@ -0,0 +1,150 @@ +// Copyright (C) 2017 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.pgm; + +import com.google.auto.value.AutoValue; +import com.google.common.base.Strings; +import com.google.gerrit.pgm.util.SiteProgram; +import com.google.gerrit.server.config.GerritServerConfig; +import com.google.gerrit.server.config.SitePaths; +import com.google.gerrit.server.schema.DataSourceProvider; +import com.google.gerrit.server.schema.JdbcAccountPatchReviewStore; +import com.google.inject.Injector; +import com.google.inject.Key; + +import org.eclipse.jgit.lib.Config; +import org.kohsuke.args4j.Option; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.List; + +/** Migrates AccountPatchReviewDb from one to another */ +public class MigrateAccountPatchReviewDb extends SiteProgram { + + @Option(name = "--sourceUrl", usage = "Url of source database") + private String sourceUrl; + + @Option(name = "--chunkSize", usage = "chunk size of fetching from source and push to target on each time") + private static long chunkSize = 100000; + + @Override + public int run() throws Exception { + SitePaths sitePaths = new SitePaths(getSitePath()); + Config fakeCfg = new Config(); + if (!Strings.isNullOrEmpty(sourceUrl)) { + System.out.println("source Url (custom): " + sourceUrl); + fakeCfg.setString("accountPatchReviewDb", null, "url", sourceUrl); + } + JdbcAccountPatchReviewStore sourceJdbcAccountPatchReviewStore = + JdbcAccountPatchReviewStore.createAccountPatchReviewStore(fakeCfg, + sitePaths); + + Injector dbInjector = createDbInjector(DataSourceProvider.Context.SINGLE_USER); + Config cfg = dbInjector.getInstance(Key.get(Config.class, GerritServerConfig.class)); + String targetUrl = cfg.getString("accountPatchReviewDb", null, "url"); + if (targetUrl == null) { + System.err.println("accountPatchReviewDb.url is null in gerrit.config"); + return 1; + } + System.out.println("target Url: " + targetUrl); + JdbcAccountPatchReviewStore targetJdbcAccountPatchReviewStore = + JdbcAccountPatchReviewStore.createAccountPatchReviewStore(cfg, sitePaths); + targetJdbcAccountPatchReviewStore.createTableIfNotExists(); + + if (!isTargetTableEmpty(targetJdbcAccountPatchReviewStore)) { + System.err.println("target table is not empty, cannot proceed"); + return 1; + } + + try (Connection sourceCon = sourceJdbcAccountPatchReviewStore.getConnection(); + Connection targetCon = targetJdbcAccountPatchReviewStore.getConnection(); + PreparedStatement sourceStmt = + sourceCon.prepareStatement( + "SELECT account_id, change_id, patch_set_id, file_name " + + "FROM account_patch_reviews " + + "LIMIT ? " + + "OFFSET ?"); + PreparedStatement targetStmt = + targetCon.prepareStatement("INSERT INTO account_patch_reviews " + + "(account_id, change_id, patch_set_id, file_name) VALUES " + + "(?, ?, ?, ?)")) { + targetCon.setAutoCommit(false); + long offset = 0; + List rows = selectRows(sourceStmt, offset); + while (!rows.isEmpty()) { + insertRows(targetCon, targetStmt, rows); + offset += rows.size(); + rows = selectRows(sourceStmt, offset); + } + } + return 0; + } + + @AutoValue + abstract static class Row { + abstract int accountId(); + abstract int changeId(); + abstract int patchSetId(); + abstract String fileName(); + } + + private static boolean isTargetTableEmpty(JdbcAccountPatchReviewStore store) + throws SQLException { + try (Connection con = store.getConnection(); + Statement s = con.createStatement(); + ResultSet r = s.executeQuery( + "SELECT COUNT(1) FROM account_patch_reviews")) { + if (r.next()) { + return r.getInt(1) == 0; + } + return true; + } + } + + private static List selectRows(PreparedStatement stmt, long offset) + throws SQLException { + List results = new ArrayList<>(); + stmt.setLong(1, chunkSize); + stmt.setLong(2, offset); + try (ResultSet rs = stmt.executeQuery()) { + while (rs.next()) { + results.add(new AutoValue_MigrateAccountPatchReviewDb_Row( + rs.getInt("account_id"), + rs.getInt("change_id"), + rs.getInt("patch_set_id"), + rs.getString("file_name"))); + } + } + return results; + } + + private static void insertRows(Connection con, PreparedStatement stmt, + List rows) throws SQLException { + for (Row r : rows) { + stmt.setLong(1, r.accountId()); + stmt.setLong(2, r.changeId()); + stmt.setLong(3, r.patchSetId()); + stmt.setString(4, r.fileName()); + stmt.addBatch(); + } + stmt.executeBatch(); + con.commit(); + } +} diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/schema/H2AccountPatchReviewStore.java b/gerrit-server/src/main/java/com/google/gerrit/server/schema/H2AccountPatchReviewStore.java index 6dbe916768..d07115ccab 100644 --- a/gerrit-server/src/main/java/com/google/gerrit/server/schema/H2AccountPatchReviewStore.java +++ b/gerrit-server/src/main/java/com/google/gerrit/server/schema/H2AccountPatchReviewStore.java @@ -15,14 +15,8 @@ package com.google.gerrit.server.schema; import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Optional; -import com.google.common.collect.ImmutableSet; -import com.google.common.primitives.Ints; -import com.google.gerrit.extensions.events.LifecycleListener; import com.google.gerrit.extensions.registration.DynamicItem; import com.google.gerrit.lifecycle.LifecycleModule; -import com.google.gerrit.reviewdb.client.Account; -import com.google.gerrit.reviewdb.client.PatchSet; import com.google.gerrit.server.change.AccountPatchReviewStore; import com.google.gerrit.server.config.GerritServerConfig; import com.google.gerrit.server.config.SitePaths; @@ -31,35 +25,12 @@ import com.google.gwtorm.server.OrmException; import com.google.inject.Inject; import com.google.inject.Singleton; -import org.apache.commons.dbcp.BasicDataSource; import org.eclipse.jgit.lib.Config; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.PreparedStatement; -import java.sql.ResultSet; import java.sql.SQLException; -import java.sql.Statement; -import java.util.Collection; - -import javax.sql.DataSource; @Singleton -public class H2AccountPatchReviewStore - implements AccountPatchReviewStore, LifecycleListener { - private static final Logger log = - LoggerFactory.getLogger(H2AccountPatchReviewStore.class); - - public static class Module extends LifecycleModule { - @Override - protected void configure() { - DynamicItem.bind(binder(), AccountPatchReviewStore.class) - .to(H2AccountPatchReviewStore.class); - listener().to(H2AccountPatchReviewStore.class); - } - } +public class H2AccountPatchReviewStore extends JdbcAccountPatchReviewStore { @VisibleForTesting public static class InMemoryModule extends LifecycleModule { @@ -72,16 +43,10 @@ public class H2AccountPatchReviewStore } } - private final DataSource ds; - @Inject H2AccountPatchReviewStore(@GerritServerConfig Config cfg, SitePaths sitePaths) { - this.ds = createDataSource(H2.appendUrlOptions(cfg, getUrl(sitePaths))); - } - - public static String getUrl(SitePaths sitePaths) { - return H2.createUrl(sitePaths.db_dir.resolve("account_patch_reviews")); + super(cfg, sitePaths); } /** @@ -93,196 +58,11 @@ public class H2AccountPatchReviewStore // DB_CLOSE_DELAY=-1: By default the content of an in-memory H2 database is // lost at the moment the last connection is closed. This option keeps the // content as long as the vm lives. - this.ds = createDataSource( - "jdbc:h2:mem:account_patch_reviews;DB_CLOSE_DELAY=-1"); - } - - private static DataSource createDataSource(String url) { - BasicDataSource datasource = new BasicDataSource(); - datasource.setDriverClassName("org.h2.Driver"); - datasource.setUrl(url); - datasource.setMaxActive(50); - datasource.setMinIdle(4); - datasource.setMaxIdle(16); - long evictIdleTimeMs = 1000 * 60; - datasource.setMinEvictableIdleTimeMillis(evictIdleTimeMs); - datasource.setTimeBetweenEvictionRunsMillis(evictIdleTimeMs / 2); - return datasource; + super(createDataSource("jdbc:h2:mem:account_patch_reviews;DB_CLOSE_DELAY=-1")); } @Override - public void start() { - try { - createTableIfNotExists(); - } catch (OrmException e) { - log.error("Failed to create table to store account patch reviews", e); - } - } - - public static void createTableIfNotExists(String url) throws OrmException { - try (Connection con = DriverManager.getConnection(url); - Statement stmt = con.createStatement()) { - doCreateTable(stmt); - } catch (SQLException e) { - throw convertError("create", e); - } - } - - private void createTableIfNotExists() throws OrmException { - try (Connection con = ds.getConnection(); - Statement stmt = con.createStatement()) { - doCreateTable(stmt); - } catch (SQLException e) { - throw convertError("create", e); - } - } - - private static void doCreateTable(Statement stmt) throws SQLException { - stmt.executeUpdate( - "CREATE TABLE IF NOT EXISTS account_patch_reviews (" - + "account_id INTEGER DEFAULT 0 NOT NULL, " - + "change_id INTEGER DEFAULT 0 NOT NULL, " - + "patch_set_id INTEGER DEFAULT 0 NOT NULL, " - + "file_name VARCHAR(4096) DEFAULT '' NOT NULL, " - + "CONSTRAINT primary_key_account_patch_reviews " - + "PRIMARY KEY (account_id, change_id, patch_set_id, file_name)" - + ")"); - } - - public static void dropTableIfExists(String url) throws OrmException { - try (Connection con = DriverManager.getConnection(url); - Statement stmt = con.createStatement()) { - stmt.executeUpdate("DROP TABLE IF EXISTS account_patch_reviews"); - } catch (SQLException e) { - throw convertError("create", e); - } - } - - @Override - public void stop() { - } - - @Override - public boolean markReviewed(PatchSet.Id psId, Account.Id accountId, - String path) throws OrmException { - try (Connection con = ds.getConnection(); - PreparedStatement stmt = - con.prepareStatement("INSERT INTO account_patch_reviews " - + "(account_id, change_id, patch_set_id, file_name) VALUES " - + "(?, ?, ?, ?)")) { - stmt.setInt(1, accountId.get()); - stmt.setInt(2, psId.getParentKey().get()); - stmt.setInt(3, psId.get()); - stmt.setString(4, path); - stmt.executeUpdate(); - return true; - } catch (SQLException e) { - OrmException ormException = convertError("insert", e); - if (ormException instanceof OrmDuplicateKeyException) { - return false; - } - throw ormException; - } - } - - @Override - public void markReviewed(PatchSet.Id psId, Account.Id accountId, - Collection paths) throws OrmException { - if (paths == null || paths.isEmpty()) { - return; - } - - try (Connection con = ds.getConnection(); - PreparedStatement stmt = - con.prepareStatement("INSERT INTO account_patch_reviews " - + "(account_id, change_id, patch_set_id, file_name) VALUES " - + "(?, ?, ?, ?)")) { - for (String path : paths) { - stmt.setInt(1, accountId.get()); - stmt.setInt(2, psId.getParentKey().get()); - stmt.setInt(3, psId.get()); - stmt.setString(4, path); - stmt.addBatch(); - } - stmt.executeBatch(); - } catch (SQLException e) { - OrmException ormException = convertError("insert", e); - if (ormException instanceof OrmDuplicateKeyException) { - return; - } - throw ormException; - } - } - - @Override - public void clearReviewed(PatchSet.Id psId, Account.Id accountId, String path) - throws OrmException { - try (Connection con = ds.getConnection(); - PreparedStatement stmt = - con.prepareStatement("DELETE FROM account_patch_reviews " - + "WHERE account_id = ? AND change_id = ? AND " - + "patch_set_id = ? AND file_name = ?")) { - stmt.setInt(1, accountId.get()); - stmt.setInt(2, psId.getParentKey().get()); - stmt.setInt(3, psId.get()); - stmt.setString(4, path); - stmt.executeUpdate(); - } catch (SQLException e) { - throw convertError("delete", e); - } - } - - @Override - public void clearReviewed(PatchSet.Id psId) throws OrmException { - try (Connection con = ds.getConnection(); - PreparedStatement stmt = - con.prepareStatement("DELETE FROM account_patch_reviews " - + "WHERE change_id = ? AND patch_set_id = ?")) { - stmt.setInt(1, psId.getParentKey().get()); - stmt.setInt(2, psId.get()); - stmt.executeUpdate(); - } catch (SQLException e) { - throw convertError("delete", e); - } - } - - @Override - public Optional findReviewed(PatchSet.Id psId, - Account.Id accountId) throws OrmException { - try (Connection con = ds.getConnection(); - PreparedStatement stmt = - con.prepareStatement( - "SELECT patch_set_id, file_name FROM account_patch_reviews APR1 " - + "WHERE account_id = ? AND change_id = ? AND patch_set_id = " - + "(SELECT MAX(patch_set_id) FROM account_patch_reviews APR2 WHERE " - + "APR1.account_id = APR2.account_id " - + "AND APR1.change_id = APR2.change_id " - + "AND patch_set_id <= ?)")) { - stmt.setInt(1, accountId.get()); - stmt.setInt(2, psId.getParentKey().get()); - stmt.setInt(3, psId.get()); - try (ResultSet rs = stmt.executeQuery()) { - if (rs.next()) { - PatchSet.Id id = new PatchSet.Id(psId.getParentKey(), - rs.getInt("PATCH_SET_ID")); - ImmutableSet.Builder builder = ImmutableSet.builder(); - do { - builder.add(rs.getString("FILE_NAME")); - } while (rs.next()); - - return Optional.of( - AccountPatchReviewStore.PatchSetWithReviewedFiles.create( - id, builder.build())); - } - - return Optional.absent(); - } - } catch (SQLException e) { - throw convertError("select", e); - } - } - - public static OrmException convertError(String op, SQLException err) { + public OrmException convertError(String op, SQLException err) { switch (getSQLStateInt(err)) { case 23001: // UNIQUE CONSTRAINT VIOLATION case 23505: // DUPLICATE_KEY_1 @@ -295,23 +75,4 @@ public class H2AccountPatchReviewStore return new OrmException(op + " failure on account_patch_reviews", err); } } - - private static String getSQLState(SQLException err) { - String ec; - SQLException next = err; - do { - ec = next.getSQLState(); - next = next.getNextException(); - } while (ec == null && next != null); - return ec; - } - - private static int getSQLStateInt(SQLException err) { - String s = getSQLState(err); - if (s != null) { - Integer i = Ints.tryParse(s); - return i != null ? i : -1; - } - return 0; - } } diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/schema/JdbcAccountPatchReviewStore.java b/gerrit-server/src/main/java/com/google/gerrit/server/schema/JdbcAccountPatchReviewStore.java new file mode 100644 index 0000000000..c537f9e610 --- /dev/null +++ b/gerrit-server/src/main/java/com/google/gerrit/server/schema/JdbcAccountPatchReviewStore.java @@ -0,0 +1,325 @@ +// Copyright (C) 2016 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.server.schema; + +import com.google.common.base.Optional; +import com.google.common.collect.ImmutableSet; +import com.google.common.primitives.Ints; +import com.google.gerrit.extensions.events.LifecycleListener; +import com.google.gerrit.extensions.registration.DynamicItem; +import com.google.gerrit.lifecycle.LifecycleModule; +import com.google.gerrit.reviewdb.client.Account; +import com.google.gerrit.reviewdb.client.PatchSet; +import com.google.gerrit.server.change.AccountPatchReviewStore; +import com.google.gerrit.server.config.GerritServerConfig; +import com.google.gerrit.server.config.SitePaths; +import com.google.gwtorm.server.OrmDuplicateKeyException; +import com.google.gwtorm.server.OrmException; + +import org.apache.commons.dbcp.BasicDataSource; +import org.eclipse.jgit.lib.Config; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Collection; + +import javax.sql.DataSource; + +public abstract class JdbcAccountPatchReviewStore + implements AccountPatchReviewStore, LifecycleListener { + private static final Logger log = + LoggerFactory.getLogger(JdbcAccountPatchReviewStore.class); + + public static class Module extends LifecycleModule { + private final Config cfg; + + public Module(Config cfg) { + this.cfg = cfg; + } + + @Override + protected void configure() { + String url = cfg.getString("accountPatchReviewDb", null, "url"); + if (url == null || url.contains("h2")) { + DynamicItem.bind(binder(), AccountPatchReviewStore.class) + .to(H2AccountPatchReviewStore.class); + listener().to(H2AccountPatchReviewStore.class); + } else if (url.contains("postgresql")) { + DynamicItem.bind(binder(), AccountPatchReviewStore.class) + .to(PostgresqlAccountPatchReviewStore.class); + listener().to(PostgresqlAccountPatchReviewStore.class); + } else if (url.contains("mysql")) { + DynamicItem.bind(binder(), AccountPatchReviewStore.class) + .to(MysqlAccountPatchReviewStore.class); + listener().to(MysqlAccountPatchReviewStore.class); + } else { + throw new IllegalArgumentException( + "unsupported driver type for account patch reviews db: " + url); + } + } + } + + private final DataSource ds; + + public static JdbcAccountPatchReviewStore createAccountPatchReviewStore( + Config cfg, SitePaths sitePaths) { + String url = cfg.getString("accountPatchReviewDb", null, "url"); + if (url == null || url.contains("h2")) { + return new H2AccountPatchReviewStore(cfg, sitePaths); + } else if (url.contains("postgresql")) { + return new PostgresqlAccountPatchReviewStore(cfg, sitePaths); + } else if (url.contains("mysql")) { + return new MysqlAccountPatchReviewStore(cfg, sitePaths); + } else { + throw new IllegalArgumentException( + "unsupported driver type for account patch reviews db: " + url); + } + } + + protected JdbcAccountPatchReviewStore(Config cfg, + SitePaths sitePaths) { + this.ds = createDataSource(getUrl(cfg, sitePaths)); + } + + protected JdbcAccountPatchReviewStore(DataSource ds) { + this.ds = ds; + } + + private static String getUrl(@GerritServerConfig Config cfg, + SitePaths sitePaths) { + String url = cfg.getString("accountPatchReviewDb", null, "url"); + if (url == null) { + return H2.createUrl(sitePaths.db_dir.resolve("account_patch_reviews")); + } + return url; + } + + protected static DataSource createDataSource(String url) { + BasicDataSource datasource = new BasicDataSource(); + if (url.contains("postgresql")) { + datasource.setDriverClassName("org.postgresql.Driver"); + } else if (url.contains("h2")) { + datasource.setDriverClassName("org.h2.Driver"); + } else if (url.contains("mysql")) { + datasource.setDriverClassName("com.mysql.jdbc.Driver"); + } + datasource.setUrl(url); + datasource.setMaxActive(50); + datasource.setMinIdle(4); + datasource.setMaxIdle(16); + long evictIdleTimeMs = 1000 * 60; + datasource.setMinEvictableIdleTimeMillis(evictIdleTimeMs); + datasource.setTimeBetweenEvictionRunsMillis(evictIdleTimeMs / 2); + return datasource; + } + + @Override + public void start() { + try { + createTableIfNotExists(); + } catch (OrmException e) { + log.error("Failed to create table to store account patch reviews", e); + } + } + + public Connection getConnection() throws SQLException { + return ds.getConnection(); + } + + public void createTableIfNotExists() throws OrmException { + try (Connection con = ds.getConnection(); + Statement stmt = con.createStatement()) { + doCreateTable(stmt); + } catch (SQLException e) { + throw convertError("create", e); + } + } + + private static void doCreateTable(Statement stmt) throws SQLException { + stmt.executeUpdate( + "CREATE TABLE IF NOT EXISTS account_patch_reviews (" + + "account_id INTEGER DEFAULT 0 NOT NULL, " + + "change_id INTEGER DEFAULT 0 NOT NULL, " + + "patch_set_id INTEGER DEFAULT 0 NOT NULL, " + + "file_name VARCHAR(4096) DEFAULT '' NOT NULL, " + + "CONSTRAINT primary_key_account_patch_reviews " + + "PRIMARY KEY (account_id, change_id, patch_set_id, file_name)" + + ")"); + } + + public void dropTableIfExists() throws OrmException { + try (Connection con = ds.getConnection(); + Statement stmt = con.createStatement()) { + stmt.executeUpdate("DROP TABLE IF EXISTS account_patch_reviews"); + } catch (SQLException e) { + throw convertError("create", e); + } + } + + @Override + public void stop() { + } + + @Override + public boolean markReviewed(PatchSet.Id psId, Account.Id accountId, + String path) throws OrmException { + try (Connection con = ds.getConnection(); + PreparedStatement stmt = + con.prepareStatement("INSERT INTO account_patch_reviews " + + "(account_id, change_id, patch_set_id, file_name) VALUES " + + "(?, ?, ?, ?)")) { + stmt.setInt(1, accountId.get()); + stmt.setInt(2, psId.getParentKey().get()); + stmt.setInt(3, psId.get()); + stmt.setString(4, path); + stmt.executeUpdate(); + return true; + } catch (SQLException e) { + OrmException ormException = convertError("insert", e); + if (ormException instanceof OrmDuplicateKeyException) { + return false; + } + throw ormException; + } + } + + @Override + public void markReviewed(PatchSet.Id psId, Account.Id accountId, + Collection paths) throws OrmException { + if (paths == null || paths.isEmpty()) { + return; + } + + try (Connection con = ds.getConnection(); + PreparedStatement stmt = + con.prepareStatement("INSERT INTO account_patch_reviews " + + "(account_id, change_id, patch_set_id, file_name) VALUES " + + "(?, ?, ?, ?)")) { + for (String path : paths) { + stmt.setInt(1, accountId.get()); + stmt.setInt(2, psId.getParentKey().get()); + stmt.setInt(3, psId.get()); + stmt.setString(4, path); + stmt.addBatch(); + } + stmt.executeBatch(); + } catch (SQLException e) { + OrmException ormException = convertError("insert", e); + if (ormException instanceof OrmDuplicateKeyException) { + return; + } + throw ormException; + } + } + + @Override + public void clearReviewed(PatchSet.Id psId, Account.Id accountId, String path) + throws OrmException { + try (Connection con = ds.getConnection(); + PreparedStatement stmt = + con.prepareStatement("DELETE FROM account_patch_reviews " + + "WHERE account_id = ? AND change_id = ? AND " + + "patch_set_id = ? AND file_name = ?")) { + stmt.setInt(1, accountId.get()); + stmt.setInt(2, psId.getParentKey().get()); + stmt.setInt(3, psId.get()); + stmt.setString(4, path); + stmt.executeUpdate(); + } catch (SQLException e) { + throw convertError("delete", e); + } + } + + @Override + public void clearReviewed(PatchSet.Id psId) throws OrmException { + try (Connection con = ds.getConnection(); + PreparedStatement stmt = + con.prepareStatement("DELETE FROM account_patch_reviews " + + "WHERE change_id = ? AND patch_set_id = ?")) { + stmt.setInt(1, psId.getParentKey().get()); + stmt.setInt(2, psId.get()); + stmt.executeUpdate(); + } catch (SQLException e) { + throw convertError("delete", e); + } + } + + @Override + public Optional findReviewed(PatchSet.Id psId, + Account.Id accountId) throws OrmException { + try (Connection con = ds.getConnection(); + PreparedStatement stmt = + con.prepareStatement( + "SELECT patch_set_id, file_name FROM account_patch_reviews APR1 " + + "WHERE account_id = ? AND change_id = ? AND patch_set_id = " + + "(SELECT MAX(patch_set_id) FROM account_patch_reviews APR2 WHERE " + + "APR1.account_id = APR2.account_id " + + "AND APR1.change_id = APR2.change_id " + + "AND patch_set_id <= ?)")) { + stmt.setInt(1, accountId.get()); + stmt.setInt(2, psId.getParentKey().get()); + stmt.setInt(3, psId.get()); + try (ResultSet rs = stmt.executeQuery()) { + if (rs.next()) { + PatchSet.Id id = new PatchSet.Id(psId.getParentKey(), + rs.getInt("patch_set_id")); + ImmutableSet.Builder builder = ImmutableSet.builder(); + do { + builder.add(rs.getString("file_name")); + } while (rs.next()); + + return Optional.of( + AccountPatchReviewStore.PatchSetWithReviewedFiles.create( + id, builder.build())); + } + + return Optional.absent(); + } + } catch (SQLException e) { + throw convertError("select", e); + } + } + + public OrmException convertError(String op, SQLException err) { + if (err.getCause() == null && err.getNextException() != null) { + err.initCause(err.getNextException()); + } + return new OrmException(op + " failure on account_patch_reviews", err); + } + + private static String getSQLState(SQLException err) { + String ec; + SQLException next = err; + do { + ec = next.getSQLState(); + next = next.getNextException(); + } while (ec == null && next != null); + return ec; + } + + protected static int getSQLStateInt(SQLException err) { + String s = getSQLState(err); + if (s != null) { + Integer i = Ints.tryParse(s); + return i != null ? i : -1; + } + return 0; + } +} diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/schema/MysqlAccountPatchReviewStore.java b/gerrit-server/src/main/java/com/google/gerrit/server/schema/MysqlAccountPatchReviewStore.java new file mode 100644 index 0000000000..7cf28ffee5 --- /dev/null +++ b/gerrit-server/src/main/java/com/google/gerrit/server/schema/MysqlAccountPatchReviewStore.java @@ -0,0 +1,53 @@ +// Copyright (C) 2017 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.server.schema; + +import com.google.gerrit.server.config.GerritServerConfig; +import com.google.gerrit.server.config.SitePaths; +import com.google.gwtorm.server.OrmDuplicateKeyException; +import com.google.gwtorm.server.OrmException; +import com.google.inject.Inject; +import com.google.inject.Singleton; + +import org.eclipse.jgit.lib.Config; + +import java.sql.SQLException; + +@Singleton +public class MysqlAccountPatchReviewStore extends JdbcAccountPatchReviewStore { + + @Inject + MysqlAccountPatchReviewStore(@GerritServerConfig Config cfg, + SitePaths sitePaths) { + super(cfg, sitePaths); + } + + @Override + public OrmException convertError(String op, SQLException err) { + switch (getSQLStateInt(err)) { + case 1022: // ER_DUP_KEY + case 1062: // ER_DUP_ENTRY + case 1169: // ER_DUP_UNIQUE; + return new OrmDuplicateKeyException("ACCOUNT_PATCH_REVIEWS", err); + + + default: + if (err.getCause() == null && err.getNextException() != null) { + err.initCause(err.getNextException()); + } + return new OrmException(op + " failure on ACCOUNT_PATCH_REVIEWS", err); + } + } +} diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/schema/PostgresqlAccountPatchReviewStore.java b/gerrit-server/src/main/java/com/google/gerrit/server/schema/PostgresqlAccountPatchReviewStore.java new file mode 100644 index 0000000000..c264c68159 --- /dev/null +++ b/gerrit-server/src/main/java/com/google/gerrit/server/schema/PostgresqlAccountPatchReviewStore.java @@ -0,0 +1,54 @@ +// Copyright (C) 2017 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.server.schema; + +import com.google.gerrit.server.config.GerritServerConfig; +import com.google.gerrit.server.config.SitePaths; +import com.google.gwtorm.server.OrmDuplicateKeyException; +import com.google.gwtorm.server.OrmException; +import com.google.inject.Inject; +import com.google.inject.Singleton; + +import org.eclipse.jgit.lib.Config; + +import java.sql.SQLException; + +@Singleton +public class PostgresqlAccountPatchReviewStore extends JdbcAccountPatchReviewStore { + + @Inject + PostgresqlAccountPatchReviewStore(@GerritServerConfig Config cfg, + SitePaths sitePaths) { + super(cfg, sitePaths); + } + + @Override + public OrmException convertError(String op, SQLException err) { + switch (getSQLStateInt(err)) { + case 23505: // DUPLICATE_KEY_1 + return new OrmDuplicateKeyException("ACCOUNT_PATCH_REVIEWS", err); + + case 23514: // CHECK CONSTRAINT VIOLATION + case 23503: // FOREIGN KEY CONSTRAINT VIOLATION + case 23502: // NOT NULL CONSTRAINT VIOLATION + case 23001: // RESTRICT VIOLATION + default: + if (err.getCause() == null && err.getNextException() != null) { + err.initCause(err.getNextException()); + } + return new OrmException(op + " failure on ACCOUNT_PATCH_REVIEWS", err); + } + } +} diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/schema/SchemaUpdater.java b/gerrit-server/src/main/java/com/google/gerrit/server/schema/SchemaUpdater.java index 0b7e8b0bab..24022e91d2 100644 --- a/gerrit-server/src/main/java/com/google/gerrit/server/schema/SchemaUpdater.java +++ b/gerrit-server/src/main/java/com/google/gerrit/server/schema/SchemaUpdater.java @@ -22,6 +22,7 @@ import com.google.gerrit.server.GerritPersonIdent; import com.google.gerrit.server.config.AllProjectsName; import com.google.gerrit.server.config.AllUsersName; import com.google.gerrit.server.config.AnonymousCowardName; +import com.google.gerrit.server.config.GerritServerConfig; import com.google.gerrit.server.config.SitePaths; import com.google.gerrit.server.git.GitRepositoryManager; import com.google.gwtorm.server.OrmException; @@ -35,6 +36,7 @@ import com.google.inject.Provider; import com.google.inject.Stage; import org.eclipse.jgit.errors.ConfigInvalidException; +import org.eclipse.jgit.lib.Config; import org.eclipse.jgit.lib.PersonIdent; import java.io.IOException; @@ -71,6 +73,7 @@ public class SchemaUpdater { for (Key k : new Key[]{ Key.get(PersonIdent.class, GerritPersonIdent.class), Key.get(String.class, AnonymousCowardName.class), + Key.get(Config.class, GerritServerConfig.class), }) { rebind(parent, k); } diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/schema/Schema_127.java b/gerrit-server/src/main/java/com/google/gerrit/server/schema/Schema_127.java index 783cce6603..cc2b0b2034 100644 --- a/gerrit-server/src/main/java/com/google/gerrit/server/schema/Schema_127.java +++ b/gerrit-server/src/main/java/com/google/gerrit/server/schema/Schema_127.java @@ -15,13 +15,15 @@ package com.google.gerrit.server.schema; import com.google.gerrit.reviewdb.server.ReviewDb; +import com.google.gerrit.server.config.GerritServerConfig; import com.google.gerrit.server.config.SitePaths; import com.google.gwtorm.server.OrmException; import com.google.inject.Inject; import com.google.inject.Provider; +import org.eclipse.jgit.lib.Config; + import java.sql.Connection; -import java.sql.DriverManager; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; @@ -31,19 +33,24 @@ public class Schema_127 extends SchemaVersion { private static final int MAX_BATCH_SIZE = 1000; private final SitePaths sitePaths; + private final Config cfg; @Inject - Schema_127(Provider prior, SitePaths sitePaths) { + Schema_127(Provider prior, + SitePaths sitePaths, + @GerritServerConfig Config cfg) { super(prior); this.sitePaths = sitePaths; + this.cfg = cfg; } @Override protected void migrateData(ReviewDb db, UpdateUI ui) throws OrmException { - String url = H2AccountPatchReviewStore.getUrl(sitePaths); - H2AccountPatchReviewStore.dropTableIfExists(url); - H2AccountPatchReviewStore.createTableIfNotExists(url); - try (Connection con = DriverManager.getConnection(url); + JdbcAccountPatchReviewStore jdbcAccountPatchReviewStore = + JdbcAccountPatchReviewStore.createAccountPatchReviewStore(cfg, sitePaths); + jdbcAccountPatchReviewStore.dropTableIfExists(); + jdbcAccountPatchReviewStore.createTableIfNotExists(); + try (Connection con = jdbcAccountPatchReviewStore.getConnection(); PreparedStatement stmt = con.prepareStatement("INSERT INTO account_patch_reviews " + "(account_id, change_id, patch_set_id, file_name) VALUES " @@ -69,7 +76,7 @@ public class Schema_127 extends SchemaVersion { stmt.executeBatch(); } } catch (SQLException e) { - throw H2AccountPatchReviewStore.convertError("insert", e); + throw jdbcAccountPatchReviewStore.convertError("insert", e); } } } diff --git a/gerrit-war/src/main/java/com/google/gerrit/httpd/WebAppInitializer.java b/gerrit-war/src/main/java/com/google/gerrit/httpd/WebAppInitializer.java index 16a699be7d..1120e0d79f 100644 --- a/gerrit-war/src/main/java/com/google/gerrit/httpd/WebAppInitializer.java +++ b/gerrit-war/src/main/java/com/google/gerrit/httpd/WebAppInitializer.java @@ -61,7 +61,7 @@ import com.google.gerrit.server.schema.DataSourceModule; import com.google.gerrit.server.schema.DataSourceProvider; import com.google.gerrit.server.schema.DataSourceType; import com.google.gerrit.server.schema.DatabaseModule; -import com.google.gerrit.server.schema.H2AccountPatchReviewStore; +import com.google.gerrit.server.schema.JdbcAccountPatchReviewStore; import com.google.gerrit.server.schema.SchemaModule; import com.google.gerrit.server.schema.SchemaVersionCheck; import com.google.gerrit.server.securestore.SecureStoreClassName; @@ -298,7 +298,7 @@ public class WebAppInitializer extends GuiceServletContextListener modules.add(new DropWizardMetricMaker.RestModule()); modules.add(new LogFileCompressor.Module()); modules.add(new EventBroker.Module()); - modules.add(new H2AccountPatchReviewStore.Module()); + modules.add(new JdbcAccountPatchReviewStore.Module(config)); modules.add(cfgInjector.getInstance(GitRepositoryManagerModule.class)); modules.add(new StreamEventsApiListener.Module()); modules.add(new ReceiveCommitsExecutorModule());