mostly functional REST service with h2 db.

hook mraas up to db. get and post clusters.

- Hook up to h2 database.
- add rest endpoints for get and post of clusters.
- add script to start h2 db browser-ui
- tests.

Change-Id: I8d290f23f8be3a790fa43fd82b03558495576412

add cluster serialization / deserialization test.

Change-Id: Iaf40458b61f893ac7fde8ed4a13296b7e94a1536

add openstack-java-sdk for nova operations.

Change-Id: Icdfeeeaf5cb1aff5eb9532cd0eab5944dab432b4

stub out provisioner, add properties to Cluster.

Change-Id: I52f2eb455e6642f5b1626a57c82a8b9b742ad1b3

update service bootstrap to use new constructor.

Change-Id: I37bfb0b03f81e4ed02abfa4f2185442d5acfb152

mraas/README.md

@@ -0,0 +1,27 @@
+# Overview
+
+  MapReduce as a Service
+
+
+# Prerequisites
+
+* Install openstack-sdk (this needs to be put up on a mvn server somewhere)
+
+    git clone git@github.com:echohead/openstack-java-sdk.git && cd openstack-java-sdk && mvn install
+
+# Running The Application
+
+Run MRaaS with the following commands:
+
+* To package:
+
+        mvn package
+
+* To setup the h2 database:
+
+        java -jar target/mraas-0.0.1-SNAPSHOT.jar setup dev-config.yml
+
+* To start the service:
+
+        java -jar target/mraas-0.0.1-SNAPSHOT.jar server dev-config.yml
+

mraas/db_shell

@@ -0,0 +1,2 @@
+#!/bin/sh
+java -jar ~/.m2/repository/com/h2database/h2/1.3.158/h2*.jar -web  -url jdbc:h2:~/git/MRaas/mraas/target/mraas -user mraas -password mraas 

mraas/dev-config.yml

@@ -1,7 +1,18 @@
 template: Hello, %s!
-defaultName: Stranger
+
+database:
+    driverClass: org.h2.Driver
+    user: mraas
+    password: mraas
+    url: jdbc:h2:target/mraas
 
 logging:
     console:
         enabled: true
         threshold: ALL
+
+http:
+    port: 8080
+    adminPort: 8081
+    maxThreads: 100
+    minThreads: 5

mraas/pom.xml

@@ -17,6 +17,37 @@
             <artifactId>dropwizard-db</artifactId>
             <version>0.3.1</version>
         </dependency>
+        <dependency>
+            <groupId>com.yammer.dropwizard</groupId>
+            <artifactId>dropwizard-testing</artifactId>
+            <version>0.3.1</version>
+        </dependency>
+
+        <!-- h2 database for dev / testing -->
+        <dependency>
+            <groupId>com.h2database</groupId>
+            <artifactId>h2</artifactId>
+            <version>1.3.158</version>
+        </dependency>
+
+        <!-- Required to support string templates in JDBI -->
+        <dependency>
+            <groupId>org.antlr</groupId>
+            <artifactId>stringtemplate</artifactId>
+            <version>3.2.1</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.openstack</groupId>
+            <artifactId>openstack-api</artifactId>
+            <version>0.0.1-SNAPSHOT</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.projectlombok</groupId>
+            <artifactId>lombok</artifactId>
+            <version>0.11.0</version>
+        </dependency>
     </dependencies>
     
     <properties>
@@ -73,6 +104,7 @@
                     <encoding>UTF-8</encoding>
                 </configuration>
             </plugin>
+
         </plugins>
     </build>
 </project>

mraas/src/main/java/com/hpcloud/mraas/MraasService.java

@@ -2,9 +2,15 @@ package com.hpcloud.mraas;
 
 import com.yammer.dropwizard.Service;
 import com.yammer.dropwizard.config.Environment;
+import com.yammer.dropwizard.db.Database;
+import com.yammer.dropwizard.db.DatabaseFactory;
 
+import com.hpcloud.mraas.cli.SetupDatabaseCommand;
+import com.hpcloud.mraas.db.ClusterDAO;
 import com.hpcloud.mraas.health.TemplateHealthCheck;
-import com.hpcloud.mraas.resources.HelloWorldResource;
+import com.hpcloud.mraas.resources.ClusterResource;
+import com.hpcloud.mraas.resources.ClustersResource;
+import com.hpcloud.mraas.sagas.Provisioner;
 
 public class MraasService extends Service<MraasConfiguration> {
     public static void main(String[] args) throws Exception {
@@ -13,15 +19,19 @@ public class MraasService extends Service<MraasConfiguration> {
 
     private MraasService() {
         super("mraas");
+        addCommand(new SetupDatabaseCommand());
     }
 
     @Override
     protected void initialize(MraasConfiguration configuration,
-                          Environment environment) {
-        final String template = configuration.getTemplate();
-        final String defaultName = configuration.getDefaultName();
-        environment.addResource(new HelloWorldResource(template, defaultName));
-        environment.addHealthCheck(new TemplateHealthCheck(template));
+                          Environment environment) throws ClassNotFoundException {
+        final DatabaseFactory factory = new DatabaseFactory(environment);
+        final Database db = factory.build(configuration.getDatabaseConfiguration(), "h2");
+        final ClusterDAO clusterDAO = db.onDemand(ClusterDAO.class);
+
+        environment.addResource(new ClusterResource(clusterDAO));
+        environment.addResource(new ClustersResource(clusterDAO, new Provisioner()));
+        environment.addHealthCheck(new TemplateHealthCheck("Hello, %s!"));
     }
 
 }

mraas/src/main/java/com/hpcloud/mraas/cli/SetupDatabaseCommand.java

@@ -0,0 +1,31 @@
+package com.hpcloud.mraas.cli;
+
+import com.hpcloud.mraas.MraasConfiguration;
+import com.hpcloud.mraas.db.ClusterDAO;
+import com.yammer.dropwizard.AbstractService;
+import com.yammer.dropwizard.cli.ConfiguredCommand;
+import com.yammer.dropwizard.config.Environment;
+import com.yammer.dropwizard.db.Database;
+import com.yammer.dropwizard.db.DatabaseFactory;
+import com.yammer.dropwizard.logging.Log;
+import org.apache.commons.cli.CommandLine;
+
+public class SetupDatabaseCommand extends ConfiguredCommand<MraasConfiguration> {
+
+    public SetupDatabaseCommand() {
+        super("setup", "Setup the database.");
+    }
+
+    @Override
+    protected void run(AbstractService<MraasConfiguration> service, MraasConfiguration configuration, CommandLine params) throws Exception {
+
+        final Log log = Log.forClass(SetupDatabaseCommand.class);
+        final Environment environment = new Environment(configuration, service);
+        final DatabaseFactory factory = new DatabaseFactory(environment);
+        final Database db = factory.build(configuration.getDatabaseConfiguration(), "h2");
+        final ClusterDAO clusterDAO = db.onDemand(ClusterDAO.class);
+
+        log.info("creating tables.");
+        clusterDAO.createClustersTable();
+    }
+}

mraas/src/main/java/com/hpcloud/mraas/core/Cluster.java

@@ -1,19 +1,31 @@
 package com.hpcloud.mraas.core;
 
-public class Cluster {
-    private final String id;
-    private final int numNodes;
+import com.yammer.dropwizard.json.JsonSnakeCase;
+import org.codehaus.jackson.annotate.JsonProperty;
+import lombok.Data;
 
-    public Cluster(String id, int numNodes) {
+@JsonSnakeCase
+@Data
+public class Cluster {
+    @JsonProperty
+    private Long id;
+    @JsonProperty
+    private Integer numNodes;
+    @JsonProperty
+    private String authUrl;
+    @JsonProperty
+    private String userName;
+    @JsonProperty
+    private String password;
+    @JsonProperty
+    private String tenant;
+
+
+    public Cluster() {}
+
+    public Cluster(Long id, int numNodes) {
         this.id = id;
         this.numNodes = numNodes;
     }
 
-    public String getId() {
-        return id;
-    }
-
-    public int getNumNodes() {
-        return numNodes;
-    }
 }

mraas/src/main/java/com/hpcloud/mraas/db/ClusterDAO.java

@@ -0,0 +1,31 @@
+package com.hpcloud.mraas.db;
+
+import com.hpcloud.mraas.core.Cluster;
+import com.google.common.collect.ImmutableList;
+import org.skife.jdbi.v2.sqlobject.Bind;
+import org.skife.jdbi.v2.sqlobject.BindBean;
+import org.skife.jdbi.v2.sqlobject.GetGeneratedKeys;
+import org.skife.jdbi.v2.sqlobject.SqlQuery;
+import org.skife.jdbi.v2.sqlobject.SqlUpdate;
+import org.skife.jdbi.v2.sqlobject.customizers.RegisterMapperFactory;
+import org.skife.jdbi.v2.sqlobject.stringtemplate.ExternalizedSqlViaStringTemplate3;
+import org.skife.jdbi.v2.tweak.BeanMapperFactory;
+
+@ExternalizedSqlViaStringTemplate3
+@RegisterMapperFactory(BeanMapperFactory.class)
+public interface ClusterDAO {
+
+    @SqlUpdate
+    void createClustersTable();
+
+    @SqlQuery
+    Cluster findById(@Bind("id") long id);
+
+    @SqlUpdate
+    @GetGeneratedKeys
+    long create(@BindBean Cluster cluster);
+
+    @SqlQuery
+    ImmutableList<Cluster> findAll();
+}
+

mraas/src/main/java/com/hpcloud/mraas/nova/Client.java

@@ -0,0 +1,40 @@
+package com.hpcloud.mraas.nova;
+
+import org.openstack.client.common.*;
+import org.openstack.client.compute.AsyncServerOperation;
+import org.openstack.model.compute.NovaFlavor;
+import org.openstack.model.compute.NovaImage;
+import org.openstack.model.compute.NovaServerForCreate;
+
+public class Client {
+    private JerseyOpenstackSession session;
+
+
+    public Client(String auth_url, String username, String password, String tenant) {
+        this.session = new JerseyOpenstackSession();
+        OpenstackCredentials creds = new OpenstackCredentials(auth_url, username, password, tenant);
+        session.authenticate(creds);
+    }
+
+    public AsyncServerOperation createHost(String name, String flavorName, String imageName) {
+        NovaServerForCreate request = new NovaServerForCreate();
+        request.setName(name);
+        request.setFlavorRef(flavorByName(flavorName));
+        request.setImageRef(imageByName(imageName));
+        return session.getComputeClient().createServer(request);
+    }
+
+    public String imageByName(String name) {
+        for (NovaImage i : session.getComputeClient().root().images().list()) {
+            if (i.getName().equals(name)) return i.getId();
+        }
+        return null;
+    }
+
+    public String flavorByName(String name) {
+        for (NovaFlavor f : session.getComputeClient().root().flavors().list()) {
+            if (f.getName().equals(name)) return f.getId();
+        }
+        return null;
+    }
+}

mraas/src/main/java/com/hpcloud/mraas/resources/ClusterResource.java

@@ -1,30 +1,31 @@
 package com.hpcloud.mraas.resources;
 
 import com.hpcloud.mraas.core.Cluster;
+import com.hpcloud.mraas.db.ClusterDAO;
+
 import com.google.common.base.Optional;
 import com.yammer.metrics.annotation.Timed;
 
 import javax.ws.rs.GET;
 import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
 import javax.ws.rs.Produces;
 import javax.ws.rs.QueryParam;
 import javax.ws.rs.core.MediaType;
 
 
-@Path("/cluster")
+@Path("/cluster/{id}")
 @Produces(MediaType.APPLICATION_JSON)
 public class ClusterResource {
-    private final String name;
-    private final String numNodes;
+    private final ClusterDAO store;
 
-    public ClusterResource(String name, String numNodes) {
-        this.name = name;
-        this.numNodes = numNodes;
+    public ClusterResource(ClusterDAO store) {
+        this.store = store;
     }
 
     @GET
     @Timed
-    public Cluster getCluster() {
-        return new Cluster("foo", 2);
+    public Cluster getCluster(@PathParam("id") long id) {
+        return store.findById(id);
     }
 }

mraas/src/main/java/com/hpcloud/mraas/resources/ClustersResource.java

@@ -0,0 +1,37 @@
+package com.hpcloud.mraas.resources;
+
+import com.hpcloud.mraas.core.Cluster;
+import com.hpcloud.mraas.db.ClusterDAO;
+import com.hpcloud.mraas.sagas.Provisioner;
+
+import com.google.common.base.Optional;
+import com.yammer.metrics.annotation.Timed;
+
+import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
+import javax.ws.rs.POST;
+import javax.ws.rs.Produces;
+import javax.ws.rs.QueryParam;
+import javax.ws.rs.core.MediaType;
+
+
+@Path("/clusters")
+@Produces(MediaType.APPLICATION_JSON)
+public class ClustersResource {
+    private final ClusterDAO store;
+    private final Provisioner provisioner;
+
+    public ClustersResource(ClusterDAO store, Provisioner provisioner) {
+        this.store = store;
+        this.provisioner = provisioner;
+    }
+
+    @POST
+    @Timed
+    public Cluster createCluster(Cluster cluster) {
+        final long clusterId = store.create(cluster);
+        Cluster newCluster = store.findById(clusterId);
+        provisioner.provision(newCluster);
+        return newCluster;
+    }
+}

mraas/src/main/java/com/hpcloud/mraas/resources/HelloWorldResource.java

@@ -1,33 +0,0 @@
-package com.hpcloud.mraas.resources;
-
-import com.hpcloud.mraas.core.Saying;
-import com.google.common.base.Optional;
-import com.yammer.metrics.annotation.Timed;
-
-import javax.ws.rs.GET;
-import javax.ws.rs.Path;
-import javax.ws.rs.Produces;
-import javax.ws.rs.QueryParam;
-import javax.ws.rs.core.MediaType;
-import java.util.concurrent.atomic.AtomicLong;
-
-@Path("/hello-world")
-@Produces(MediaType.APPLICATION_JSON)
-public class HelloWorldResource {
-    private final String template;
-    private final String defaultName;
-    private final AtomicLong counter;
-
-    public HelloWorldResource(String template, String defaultName) {
-        this.template = template;
-        this.defaultName = defaultName;
-        this.counter = new AtomicLong();
-    }
-
-    @GET
-    @Timed
-    public Saying sayHello(@QueryParam("name") Optional<String> name) {
-        return new Saying(counter.incrementAndGet(),
-                          String.format(template, name.or(defaultName)));
-    }
-}

mraas/src/main/java/com/hpcloud/mraas/sagas/Provisioner.java

@@ -0,0 +1,11 @@
+package com.hpcloud.mraas.sagas;
+
+import com.hpcloud.mraas.core.Cluster;
+import com.hpcloud.mraas.nova.Client;
+
+public class Provisioner {
+
+    public static void provision(Cluster cluster) {
+
+    }
+}

mraas/src/main/resources/com/hpcloud/mraas/db/ClusterDAO.sql.stg

@@ -0,0 +1,18 @@
+group ClusterDAO;
+
+createClustersTable() ::= <<
+  create table clusters (id Serial primary key, numNodes int)
+>>
+
+findById() ::= <<
+  select id, numNodes from clusters where id= :id
+>>
+
+
+create() ::= <<
+  insert into clusters(id, numNodes) values (:id, :numNodes)
+>>
+
+findAll() ::= <<
+  select id, numNodes from clusters
+>>

mraas/src/test/java/com/hpcloud/mraas/tests/core/ClusterTest.java

@@ -0,0 +1,40 @@
+package com.hpcloud.mraas.tests.core;
+
+import org.codehaus.jackson.type.TypeReference;
+import org.junit.Before;
+import org.junit.Test;
+import static com.yammer.dropwizard.testing.JsonHelpers.*;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.equalTo;
+import static org.junit.Assert.assertThat;
+
+import com.hpcloud.mraas.core.Cluster;
+
+
+public class ClusterTest {
+    private Cluster cluster;
+
+    @Before
+    public void initialize() {
+        cluster = new Cluster(new Long(1), 20);
+        cluster.setAuthUrl("http://foo");
+        cluster.setUserName("foo");
+        cluster.setPassword("bar");
+        cluster.setTenant("thetenant");
+    }
+
+    @Test
+    public void serializesToJSON() throws Exception {
+        assertThat("a cluster can be serialized to JSON",
+            asJson(cluster),
+            is(equalTo(jsonFixture("fixtures/cluster.json"))));
+    }
+
+    @Test
+    public void deserializesFromJSON() throws Exception {
+        assertThat("a cluster can be deserialized from JSON",
+            fromJson(jsonFixture("fixtures/cluster.json"), Cluster.class),
+            is(cluster));
+    }
+
+}

mraas/src/test/java/com/hpcloud/mraas/tests/nova/ClientTest.java

@@ -0,0 +1,18 @@
+package com.hpcloud.mraas.tests.nova;
+
+import org.junit.Test;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.equalTo;
+import static org.junit.Assert.assertThat;
+
+import com.hpcloud.mraas.nova.Client;
+
+
+public class ClientTest {
+
+    @Test
+    public void createCluster() throws Exception {
+
+    }
+
+}

mraas/src/test/java/com/hpcloud/mraas/tests/resources/ClusterResourceTest.java

@@ -0,0 +1,82 @@
+package com.hpcloud.mraas.tests.resources;
+
+import com.yammer.dropwizard.testing.ResourceTest;
+
+import com.hpcloud.mraas.core.Cluster;
+import com.hpcloud.mraas.db.ClusterDAO;
+import com.hpcloud.mraas.sagas.Provisioner;
+import com.hpcloud.mraas.resources.ClusterResource;
+import com.hpcloud.mraas.resources.ClustersResource;
+
+import org.junit.Before;
+import org.junit.Test;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import javax.ws.rs.core.MediaType;
+import org.mockito.ArgumentMatcher;
+
+import static org.hamcrest.Matchers.is;
+import static org.junit.Assert.assertThat;
+import static org.mockito.Matchers.anyLong;
+import static org.mockito.Matchers.argThat;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+
+public class ClusterResourceTest extends ResourceTest {
+    static {
+        Logger.getLogger("com.sun.jersey").setLevel(Level.OFF);
+    }
+
+    private final Cluster cluster = new Cluster(new Long(1), 20);
+    private final ClusterDAO store = mock(ClusterDAO.class);
+    private final Provisioner provisioner = mock(Provisioner.class);
+
+    @Before
+    public void initialize() {
+        cluster.setAuthUrl("http://example.com");
+        cluster.setUserName("foo");
+        cluster.setPassword("bar");
+        cluster.setTenant("tenant");
+    }
+
+    @Override
+    protected void setUpResources() {
+        when(store.findById(anyLong())).thenReturn(cluster);
+        when(store.create(argThat(new IsCluster()))).thenReturn(new Long(1));
+        addResource(new ClusterResource(store));
+        addResource(new ClustersResource(store, provisioner));
+    }
+
+    @Test
+    public void findByIdTest() throws Exception {
+        assertThat("GET requests fetch the cluster by ID",
+            client().resource("/cluster/1").get(Cluster.class),
+            is(cluster));
+        verify(store, times(1)).findById(1);
+        verify(store, never()).create(argThat(new IsCluster()));
+    }
+
+    @Test
+    public void createClusterTest() throws Exception {
+        Cluster request = new Cluster();
+        request.setNumNodes(20);
+        assertThat("POST-ing a new cluster with no id returns the new cluster",
+            client().resource("/clusters").accept(MediaType.APPLICATION_JSON_TYPE).post(Cluster.class, request),
+            is(cluster));
+        verify(store, times(1)).create(argThat(new IsCluster()));
+        verify(store, times(1)).findById(1);
+        verify(provisioner, times(1)).provision(cluster);
+    }
+
+
+}
+
+class IsCluster extends ArgumentMatcher<Cluster> {
+    public boolean matches(Object arg) {
+        return (arg instanceof Cluster);
+    }
+}

mraas/src/test/resources/fixtures/cluster.json

@@ -0,0 +1 @@
+{"id":1,"num_nodes":20,"auth_url":"http://foo","user_name":"foo","password":"bar","tenant":"thetenant"}