Import all patch data again from Git to ensure it is accurate

We've made some schema changes about how patch data is stored and
represented in the Gerrit 2 database, so we need to import all of
the patch entities over again.  This change includes a CLI tool
that can be run to re-import one or more patch set entities from
the corresponding Git commit.

Signed-off-by: Shawn O. Pearce <sop@google.com>

.gitignore

@@ -3,4 +3,5 @@
 /gerrit2_dump.sql.bz2
 /gerrit-server.jar
 /gerrit.war
+/release
 hs_err_pid*.log

Makefile

@@ -25,7 +25,6 @@ JAVA       = java
 JAVAC      = javac
 JAR        = jar
 JAVA_ARGS  = -Xmx265m
-CPIO       = cpio -pd
 GWT_OS     = unknown
 GWT_FLAGS  =
 
@@ -103,6 +102,15 @@ clean:
 	rm -rf $(WEBAPP)/classes
 	rm -rf $(WEBAPP)/www
 	rm -rf $(WEBAPP)/tomcat
+	rm -rf release
+
+release: $(ALL_LIB) $(MY_JAR)
+	rm -rf release
+	mkdir -p release/bin release/lib
+	$(foreach p,$(ALL_LIB) $(MY_JAR) $(ALL_JDBC),cp $p release/lib &&) :
+	$(foreach p,$(wildcard bin/*),\
+	  cp $p release/$p && \
+	  chmod 555 release/$p &&) :
 
 clean-h2db:
 	rm -f $(WEBAPP)/ReviewDb.*.db
@@ -208,4 +216,5 @@ $(WEBAPP)/lib/jgit.jar: .jgit_version
 
 .PHONY: all
 .PHONY: clean
-.PHONY: web web-shell web-lib
+.PHONY: release
+.PHONY: web-shell web-lib

bin/gerrit2.sh

@@ -0,0 +1,90 @@
+#!/bin/sh
+#
+# Copyright 2008 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if [ -z "$GERRIT2_HOME" ]
+then
+	GERRIT2_HOME=`which $0 2>/dev/null`
+	GERRIT2_HOME=`dirname $GERRIT2_HOME`
+	GERRIT2_HOME=`dirname $GERRIT2_HOME`
+fi
+
+if [ -z "$GERRIT2_HOME" ]
+then
+	echo >&2 "error: GERRIT2_HOME not set, cannot guess"
+	exit 1
+fi
+
+if ! [ -f "$GERRIT2_HOME/lib/gerrit-server.jar" ]
+then
+	echo >&2 "error: $GERRIT2_HOME does not have lib/gerrit-server.jar"
+	exit 1
+fi
+
+if [ -z "$GERRIT2_JAVA" ]
+then
+	GERRIT2_JAVA=java
+fi
+
+config_dir=
+case "$1" in
+--config=*)
+	config_dir=`echo "$1" | sed s/^--config=//`
+	if ! [ -f "$config_dir" ]
+	then
+		echo >&2 "error: $config_dir not found"
+		exit 1
+	fi
+	case "$config_dir" in
+	*/GerritServer.properties)
+		config_dir=`dirname "$config_dir"`
+		shift
+		;;
+	*)
+		echo >&2 "error: --config must point to GerritServer.properties"
+		exit 1
+		;;
+	esac
+	;;
+esac
+
+if [ $# = 0 ]
+then
+	echo >&2 "usage: $0 [--config=gs.prop] AppName [args]"
+	exit 1
+fi
+app=$1
+shift
+
+if [ -n "$config_dir" ]
+then
+	CLASSPATH=$config_dir
+else
+	CLASSPATH=
+fi
+
+for j in $GERRIT2_HOME/lib/*.jar
+do
+	if [ -z "$CLASSPATH" ]
+	then
+		CLASSPATH=$j
+	else
+		CLASSPATH=$CLASSPATH:$j
+	fi
+done
+export CLASSPATH
+
+umask 0022
+exec $GERRIT2_JAVA com.google.gerrit.pgm.$app "$@"

devutil/import_gerrit1.sql

@@ -1,10 +1,25 @@
 -- PostgreSQL conversion from Gerrit 1 -> Gerrit 2
 --
--- Execute this manually:
+-- Execute this from your shell:
 --
 --  psql -c 'ALTER SCHEMA public RENAME TO gerrit1' $srcdb
---  pg_dump $srcdb >D
---  psql -f D $dstdb
+--  pg_dump $srcdb | psql $dstdb
+--  psql -f devutil/import_gerrit1.sql $dstdb
+--
+-- Run the ALTER commands displayed in a psql prompt.
+--
+-- Ensure the Git repositories are where git_base_path in the
+-- system_config table says they should be.
+--
+-- Create a GerritServer.properties file for your database.
+--
+-- Run this from your shell:
+--
+--  make release
+--  psql $dstdb -tAc 'select change_id,patch_set_id from patch_sets' \
+--  | release/bin/gerrit2.sh \
+--    --config=GerritServer.properties \
+--    ReimportPatchSets
 --
 
 DELETE FROM accounts;

webapp/src/com/google/gerrit/client/reviewdb/Patch.java

@@ -132,10 +132,10 @@ public final class Patch {
   protected Patch() {
   }
 
-  public Patch(final Patch.Id newId, final ChangeType ct, final PatchType pt) {
+  public Patch(final Patch.Id newId) {
     key = newId;
-    setChangeType(ct);
-    setPatchType(pt);
+    setChangeType(ChangeType.MODIFIED);
+    setPatchType(PatchType.UNIFIED);
   }
 
   public Patch.Id getKey() {

webapp/src/com/google/gerrit/client/reviewdb/PatchSet.java

@@ -59,11 +59,6 @@ public final class PatchSet {
     key = k;
   }
 
-  public PatchSet(final PatchSet.Id k, final RevId rev) {
-    this(k);
-    revision = rev;
-  }
-
   public PatchSet.Id getKey() {
     return key;
   }
@@ -75,4 +70,8 @@ public final class PatchSet {
   public RevId getRevision() {
     return revision;
   }
+
+  public void setRevision(final RevId i) {
+    revision = i;
+  }
 }

webapp/src/com/google/gerrit/client/reviewdb/PatchSetInfo.java

@@ -58,7 +58,11 @@ public final class PatchSetInfo {
   }
 
   public void setSubject(final String s) {
-    subject = s;
+    if (s != null && s.length() > 255) {
+      subject = s.substring(0, 255);
+    } else {
+      subject = s;
+    }
   }
 
   public String getMessage() {

webapp/src/com/google/gerrit/git/GitMetaUtil.java

@@ -18,8 +18,31 @@ import org.spearce.jgit.lib.Repository;
 
 import java.io.File;
 import java.io.IOException;
+import java.util.Random;
 
 public class GitMetaUtil {
+  private static final int SLEEP_MIN = 100; // milliseconds
+  private static final int SLEEP_MAX = 2 * 60 * 1000; // milliseconds
+  private static final ThreadLocal<Random> SLEEP_RNG =
+      new ThreadLocal<Random>() {
+        @Override
+        protected Random initialValue() {
+          return new Random();
+        }
+      };
+
+  private static int waitTime() {
+    return SLEEP_MIN + SLEEP_RNG.get().nextInt(SLEEP_MAX - SLEEP_MIN);
+  }
+
+  public static void randomSleep() {
+    try {
+      Thread.sleep(waitTime());
+    } catch (InterruptedException ie) {
+      // Just let the thread continue anyway.
+    }
+  }
+
   public static boolean isGitRepository(final File gitdir) {
     return new File(gitdir, "config").isFile()
         && new File(gitdir, "HEAD").isFile()

webapp/src/com/google/gerrit/git/PatchSetImporter.java

@@ -0,0 +1,274 @@
+// Copyright 2008 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.gerrit.git;
+
+import com.google.gerrit.client.reviewdb.Patch;
+import com.google.gerrit.client.reviewdb.PatchContent;
+import com.google.gerrit.client.reviewdb.PatchSet;
+import com.google.gerrit.client.reviewdb.PatchSetInfo;
+import com.google.gerrit.client.reviewdb.RevId;
+import com.google.gerrit.client.reviewdb.ReviewDb;
+import com.google.gerrit.client.reviewdb.UserIdentity;
+import com.google.gwtorm.client.OrmException;
+import com.google.gwtorm.client.Transaction;
+
+import org.spearce.jgit.lib.Commit;
+import org.spearce.jgit.lib.Constants;
+import org.spearce.jgit.lib.ObjectId;
+import org.spearce.jgit.lib.ObjectWriter;
+import org.spearce.jgit.lib.PersonIdent;
+import org.spearce.jgit.lib.Repository;
+import org.spearce.jgit.lib.Tree;
+import org.spearce.jgit.patch.CombinedFileHeader;
+import org.spearce.jgit.patch.FileHeader;
+import org.spearce.jgit.revwalk.RevCommit;
+
+import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+import java.security.MessageDigest;
+import java.sql.Timestamp;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+/** Imports a {@link PatchSet} from a {@link Commit}. */
+public class PatchSetImporter {
+  private static final int MAX_TRIES = 10;
+  private final ReviewDb db;
+  private final Repository repo;
+  private final RevCommit src;
+  private final PatchSet dst;
+  private final boolean isNew;
+  private org.spearce.jgit.patch.Patch gitpatch;
+
+  private PatchSetInfo info;
+  private boolean infoIsNew;
+
+  private final MessageDigest contentmd = Constants.newMessageDigest();
+  private final Map<String, Patch> patchExisting = new HashMap<String, Patch>();
+  private final List<Patch> patchInsert = new ArrayList<Patch>();
+  private final List<Patch> patchUpdate = new ArrayList<Patch>();
+  private final Map<PatchContent.Key, String> content =
+      new HashMap<PatchContent.Key, String>();
+
+  public PatchSetImporter(final ReviewDb dstDb, final Repository srcRepo,
+      final RevCommit srcCommit, final PatchSet dstPatchSet,
+      final boolean isNewPatchSet) {
+    db = dstDb;
+    repo = srcRepo;
+    src = srcCommit;
+    dst = dstPatchSet;
+    isNew = isNewPatchSet;
+  }
+
+  public void run() throws IOException, OrmException {
+    gitpatch = readGitPatch();
+
+    dst.setRevision(new RevId(src.getId().name()));
+
+    if (!isNew) {
+      // If we aren't a new patch set then we need to load the existing
+      // files so we can update or delete them if there are corrections.
+      //
+      info = db.patchSetInfo().get(dst.getKey());
+      for (final Patch p : db.patches().byPatchSet(dst.getKey())) {
+        patchExisting.put(p.getFileName(), p);
+      }
+    }
+
+    importInfo();
+    for (final FileHeader fh : gitpatch.getFiles()) {
+      importFile(fh);
+    }
+
+    // Ensure all content entities exist
+    //
+    putPatchContent();
+
+    final Transaction txn = db.beginTransaction();
+    if (isNew) {
+      db.patchSets().insert(Collections.singleton(dst));
+    }
+    if (infoIsNew) {
+      db.patchSetInfo().insert(Collections.singleton(info));
+    } else {
+      db.patchSetInfo().update(Collections.singleton(info));
+    }
+    db.patches().insert(patchInsert, txn);
+    if (!isNew) {
+      db.patches().update(patchUpdate, txn);
+      db.patches().delete(patchExisting.values(), txn);
+    }
+    txn.commit();
+  }
+
+  private void importInfo() {
+    if (info == null) {
+      info = new PatchSetInfo(dst.getKey());
+      infoIsNew = true;
+    }
+
+    info.setSubject(src.getShortMessage());
+    info.setMessage(src.getFullMessage());
+    info.setAuthor(toUserIdentity(src.getAuthorIdent()));
+    info.setCommitter(toUserIdentity(src.getCommitterIdent()));
+  }
+
+  private UserIdentity toUserIdentity(final PersonIdent who) {
+    final UserIdentity u = new UserIdentity();
+    u.setName(who.getName());
+    u.setEmail(who.getEmailAddress());
+    u.setDate(new Timestamp(who.getWhen().getTime()));
+    u.setTimeZone(who.getTimeZoneOffset());
+    return u;
+  }
+
+  private void importFile(final FileHeader fh)
+      throws UnsupportedEncodingException {
+    final String path;
+    if (fh.getChangeType() == FileHeader.ChangeType.DELETE) {
+      path = fh.getOldName();
+    } else {
+      path = fh.getNewName();
+    }
+
+    Patch p = patchExisting.remove(path);
+    if (p == null) {
+      p = new Patch(new Patch.Id(dst.getKey(), path));
+      patchInsert.add(p);
+    } else {
+      p.setSourceFileName(null);
+      patchUpdate.add(p);
+    }
+
+    // Convert the ChangeType
+    //
+    if (fh.getChangeType() == FileHeader.ChangeType.ADD) {
+      p.setChangeType(Patch.ChangeType.ADD);
+
+    } else if (fh.getChangeType() == FileHeader.ChangeType.MODIFY) {
+      p.setChangeType(Patch.ChangeType.MODIFIED);
+
+    } else if (fh.getChangeType() == FileHeader.ChangeType.DELETE) {
+      p.setChangeType(Patch.ChangeType.DELETED);
+
+    } else if (fh.getChangeType() == FileHeader.ChangeType.RENAME) {
+      p.setChangeType(Patch.ChangeType.RENAMED);
+      p.setSourceFileName(fh.getOldName());
+
+    } else if (fh.getChangeType() == FileHeader.ChangeType.COPY) {
+      p.setChangeType(Patch.ChangeType.COPIED);
+      p.setSourceFileName(fh.getOldName());
+    }
+
+    // Convert the PatchType
+    //
+    if (fh instanceof CombinedFileHeader) {
+      p.setPatchType(Patch.PatchType.N_WAY);
+
+    } else if (fh.getPatchType() == FileHeader.PatchType.GIT_BINARY) {
+      p.setPatchType(Patch.PatchType.BINARY);
+
+    } else if (fh.getPatchType() == FileHeader.PatchType.BINARY) {
+      p.setPatchType(Patch.PatchType.BINARY);
+    }
+
+    // Hash the content.
+    //
+    final String contentStr = fh.getScriptText();
+    contentmd.reset();
+    contentmd.update(contentStr.getBytes("UTF-8"));
+    final PatchContent.Key contentKey =
+        new PatchContent.Key(ObjectId.fromRaw(contentmd.digest()).name());
+    content.put(contentKey, contentStr);
+    p.setContent(contentKey);
+  }
+
+  private void putPatchContent() throws OrmException {
+    OrmException contentPutError = null;
+    for (int attempts = 0; !content.isEmpty() && ++attempts < MAX_TRIES;) {
+      for (final PatchContent pc : db.patchContents().get(content.keySet())) {
+        content.remove(pc.getKey());
+      }
+
+      for (final Iterator<Map.Entry<PatchContent.Key, String>> i =
+          content.entrySet().iterator(); i.hasNext();) {
+        final Map.Entry<PatchContent.Key, String> e = i.next();
+        final PatchContent pc = new PatchContent(e.getKey(), e.getValue());
+        try {
+          db.patchContents().insert(Collections.singleton(pc));
+          i.remove();
+        } catch (OrmException err) {
+          contentPutError = err;
+        }
+      }
+
+      if (!content.isEmpty()) {
+        GitMetaUtil.randomSleep();
+      }
+    }
+    if (!content.isEmpty() && contentPutError != null) {
+      throw contentPutError;
+    }
+  }
+
+  private org.spearce.jgit.patch.Patch readGitPatch() throws IOException {
+    final List<String> args = new ArrayList<String>();
+    args.add("git");
+    args.add("--git-dir=.");
+    args.add("diff-tree");
+    args.add("-M");
+    args.add("--full-index");
+
+    switch (src.getParentCount()) {
+      case 0:
+        args.add("--unified=5");
+        args.add(new ObjectWriter(repo).writeTree(new Tree(repo)).name());
+        args.add(src.getTree().getId().name());
+        break;
+      case 1:
+        args.add("--unified=5");
+        args.add(src.getParent(0).getId().name());
+        args.add(src.getId().name());
+        break;
+      default:
+        args.add("--cc");
+        args.add(src.getId().name());
+        break;
+    }
+
+    final Process proc =
+        Runtime.getRuntime().exec(args.toArray(new String[args.size()]), null,
+            repo.getDirectory());
+    try {
+      final org.spearce.jgit.patch.Patch p = new org.spearce.jgit.patch.Patch();
+      proc.getOutputStream().close();
+      proc.getErrorStream().close();
+      p.parse(proc.getInputStream());
+      proc.getInputStream().close();
+      return p;
+    } finally {
+      try {
+        if (proc.waitFor() != 0) {
+          throw new IOException("git diff-tree exited abnormally");
+        }
+      } catch (InterruptedException ie) {
+      }
+    }
+  }
+}

webapp/src/com/google/gerrit/pgm/ReimportPatchSets.java

@@ -0,0 +1,117 @@
+// Copyright 2008 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.gerrit.pgm;
+
+import com.google.gerrit.client.reviewdb.Change;
+import com.google.gerrit.client.reviewdb.PatchSet;
+import com.google.gerrit.client.reviewdb.ReviewDb;
+import com.google.gerrit.git.InvalidRepositoryException;
+import com.google.gerrit.git.PatchSetImporter;
+import com.google.gerrit.server.GerritServer;
+import com.google.gwtjsonrpc.server.XsrfException;
+import com.google.gwtorm.client.OrmException;
+
+import org.spearce.jgit.lib.ObjectId;
+import org.spearce.jgit.lib.Repository;
+import org.spearce.jgit.revwalk.RevCommit;
+import org.spearce.jgit.revwalk.RevWalk;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.sql.SQLException;
+import java.util.ArrayList;
+
+/**
+ * Recreates PatchSet and Patch entities for the changes supplied.
+ * <p>
+ * Takes on input strings of the form <code>change_id|patch_set_id</code>, such
+ * as might be created by the following PostgreSQL database dump:
+ * 
+ * <pre>
+ *  psql reviewdb -tAc 'select change_id,patch_set_id from patch_sets'
+ * </pre>
+ * <p>
+ * For each supplied PatchSet the info and patch entities are completely updated
+ * based on the data stored in Git.
+ */
+public class ReimportPatchSets {
+  public static void main(final String[] argv) throws OrmException,
+      XsrfException, IOException {
+    final ArrayList<PatchSet.Id> todo = new ArrayList<PatchSet.Id>();
+    final BufferedReader br =
+        new BufferedReader(new InputStreamReader(System.in));
+    String line;
+    while ((line = br.readLine()) != null) {
+      final String[] idstr = line.split("\\|");
+      todo.add(new PatchSet.Id(Change.Id.fromString(idstr[0]), Integer
+          .parseInt(idstr[1])));
+    }
+
+    final GerritServer gs = GerritServer.getInstance();
+    final ReviewDb db = gs.getDatabase().open();
+    try {
+      for (int i = 0; i < todo.size(); i++) {
+        System.out.print("\rImport " + (i + 1) + " of " + todo.size() + " ...");
+        System.out.flush();
+
+        final PatchSet.Id psid = todo.get(i);
+        final PatchSet ps = db.patchSets().get(psid);
+        if (ps == null) {
+          System.out.println();
+          System.err.println("NotFound " + psid);
+          System.err.flush();
+          continue;
+        }
+
+        final Change c = db.changes().get(ps.getKey().getParentKey());
+        if (c == null) {
+          System.out.println();
+          System.err.println("Orphan " + psid);
+          System.err.flush();
+          continue;
+        }
+
+        final String projectName = c.getDest().getParentKey().get();
+        final Repository repo;
+        try {
+          repo = gs.getRepositoryCache().get(projectName);
+        } catch (InvalidRepositoryException ie) {
+          System.out.println();
+          System.err.println("NoProject " + psid);
+          System.err.println("NoProject " + ie.getMessage());
+          System.err.flush();
+          continue;
+        }
+
+        final RevWalk rw = new RevWalk(repo);
+        final RevCommit src =
+            rw.parseCommit(ObjectId.fromString(ps.getRevision().get()));
+        new PatchSetImporter(db, repo, src, ps, false).run();
+      }
+    } catch (OrmException e) {
+      e.printStackTrace();
+      if (e.getCause() instanceof SQLException) {
+        final SQLException e2 = (SQLException) e.getCause();
+        if (e2.getNextException() != null) {
+          e2.getNextException().printStackTrace();
+        }
+      }
+    } finally {
+      System.out.println();
+      db.close();
+    }
+  }
+}