S3 storage (not to be merged)

README.md

@@ -1,10 +1,14 @@
 Takipi Storage
 ==============
 
+Thanks to Moovel for developing this version supporting s3: https://github.com/moovel/takipi-storage/tree/s3-storage
+And to Atlassian for adding some extra features
+
 Build and run:
 - clone the repo
 - `cd takipi-storage`
 - `mvn compile package`
+- edit settings.yml to contain bucket, key, secret to access your s3 bucket
 - `java -jar target/takipi-storage-1.7.0.jar server settings.yml`
 
 Deploy:

docker/Dockerfile

@@ -1,21 +1,35 @@
-# Run me with docker run -v <host-path-to-storage>:/opt/takipi-storage/storage -p <host_port>:$STORAGE_PORT
-# Logs are written to <storage>/log folder and both should be persistent outside of the running container
+FROM openjdk:8-jre-slim
+LABEL maintainer="[email protected]"
 
-FROM java:8
-MAINTAINER Chen harel "https://github.com/chook"
+ARG APP_VERSION=latest
 
-ENV VERSION 1.7.0
-ENV TAR_FILENAME takipi-storage-$VERSION.tar.gz
-ENV JAR_FILENAME takipi-storage-$VERSION.jar
-ENV STORAGE_PORT 8080
+# install curl
+RUN apt-get update; apt-get install -y curl
 
-RUN wget https://s3.amazonaws.com/app-takipi-com/deploy/takipi-storage/$TAR_FILENAME
-RUN tar zxvf $TAR_FILENAME -C /tmp && \
-    mkdir -p /opt/takipi-storage/lib && \
-    cp /tmp/takipi-storage/lib/$JAR_FILENAME /opt/takipi-storage/lib
-ADD settings.yml /opt/takipi-storage
+# rootless
+RUN groupadd --gid 1000 overops
+RUN adduser --home /opt/takipi-storage --uid 1000 --gid 1000 overops
+USER 1000:1000
 
-EXPOSE $STORAGE_PORT
+# install into the /opt directory
+WORKDIR /opt
+
+# download and install the storage server
+RUN curl -sL http://app-takipi-com.s3.amazonaws.com/deployx/s3/deploy/takipi-storage/takipi-storage-${APP_VERSION}.tar.gz | tar -xvzf -
+
+RUN mkdir /opt/takipi-storage/private
+COPY --chown=1000:1000 "./private/settings.yaml" "/opt/takipi-storage/private/settings.yaml"
+
+# use mount to make settings.yaml available
+VOLUME ["/opt/takipi-storage/private"]
 
 WORKDIR /opt/takipi-storage
-CMD java -jar /opt/takipi-storage/lib/$JAR_FILENAME server settings.yml
+
+# copy the run script
+COPY --chown=1000:1000 "./scripts/run.sh" "./run.sh"
+RUN chmod +x run.sh
+
+EXPOSE 8080 8081
+
+# run the service, printing logs to stdout
+CMD ["./run.sh"]

docker/Jenkinsfile

@@ -0,0 +1,72 @@
+def imageName = 'docker/overops-storage-server-s3'
+def dockerHubImage = 'overops/storage-server-s3'
+
+pipeline {
+
+    environment {
+        registryCred = 'container-registry-build-guy'
+        dockerhubCred = 'docker-hub'
+        gitCred = 'build-guy'
+    }
+
+    parameters {
+        string(name: 'VERSION', defaultValue: 'latest', description:'Application version')
+        string(name: 'TAG', defaultValue: 'latest', description:'Image Tag to be used')
+        booleanParam(name: 'PUBLISH_TO_DOCKERHUB', defaultValue: false, description:'Flag to publish to docker-hub')        
+    }    
+
+    agent any
+    stages {
+        stage('Cloning Git') {
+            steps {
+                git([url: 'https://github.com/takipi/takipi-storage', branch: 's3-storage', credentialsId: gitCred])
+            }
+        }
+
+        stage('Build Docker Image') {
+            steps {
+                dir('docker') {
+                    script {
+                        if (params.PUBLISH_TO_DOCKERHUB) {
+                            imageName = dockerHubImage
+                        }
+
+                        dockerOptions = ('--label=storage-server-s3-pipeline  --build-arg APP_VERSION=' + params.VERSION + ' .')
+                        dockerImage = docker.build(imageName, dockerOptions)
+                    }
+                }
+            }
+        }
+
+        stage('Publish Docker Image') {
+            steps {
+                script {
+                    if (params.PUBLISH_TO_DOCKERHUB) {
+                        reg = ''
+                        cred = dockerhubCred
+                    } else {
+                        reg = env.LOCAL_DOCKER_REGISTRY_URL
+                        cred = registryCred
+                    }
+
+                    docker.withRegistry(reg, cred) {
+                        dockerImage.push()
+
+                        if (params.TAG != 'latest') {
+                            dockerImage.push(params.TAG)
+                        }
+                    }
+                }
+            }
+        }
+
+        stage('Cleanup') {
+            steps {
+                script {
+                    sh(script:"docker rmi -f \$(docker images -f label=storage-server-s3-pipeline -q)")
+                }
+                cleanWs()
+            }
+        }
+    }
+}

docker/README.md

@@ -1,6 +1,19 @@
-takipi-storage on docker
-========================
+# Deploy the Storage Server - S3 (hybrid installations, AWS S3)
 
-When running takipi-storage inside docker, make sure to persist the storage folder outside (using volumes).
+For hybrid installations, the Storage Server can be installed in your cluster.
 
-Logs are placed inside the storage folder
+This Storage Server [Dockerfile](Dockerfile) is based on the [Installing the Storage Server on AWS S3](https://doc.overops.com/docs/installing-the-storage-server-on-aws-s3) guide, with some minor modifications.
+
+For complete instructions on performing a hybrid installation, refer to the [Hybrid Installation on Linux](https://doc.overops.com/docs/linux-hybrid-installation) guide.
+
+The file `settings.yaml` must be mounted into the `/opt/takipi-storage/private` directory to run this container. An example [settings.yaml](private/settings.yaml) can be found in this repo.
+
+## Quick Start
+
+This image is on Docker Hub: [overops/storage-server-s3](https://hub.docker.com/r/overops/storage-server-s3)
+
+### Docker Quick Start
+
+```console
+docker run -d -p 8080:8080 --mount type=bind,source="$(pwd)"/private,target=/opt/takipi-storage/private overops/storage-server-s3
+```

docker/private/settings.yaml

@@ -0,0 +1,45 @@
+enableCors: true
+corsOrigins: "*"
+
+# If using attaching IAM Role to instance leave accessKey and secretKey empty.
+s3Fs:
+    bucket: <BUCKET_NAME>
+    pathPrefix: <FOLDER_IN_BUCKET>
+    credentials:
+      accessKey: <AWS_ACCESS_KEY>
+      secretKey: <AWS_SECRET_KEY>
+
+multifetch:
+  # maximum number of threads for concurrent multi-fetch. Set to zero to disable concurrent fetching.
+  concurrencyLevel: 100
+  # Recommended size >= 16777216. Set to zero to disable caching.
+  maxCacheSize: 67108864
+  enableCacheLogger: false
+  maxBatchSize: 4194304      
+
+server:
+  gzip:
+    includedMethods: [POST, GET]  
+  applicationConnectors:
+    - type: http
+      port: 8080
+# https support
+#   - type: https
+#     port: 8443
+#     keyStorePath: example.keystore
+#     keyStorePassword: example
+#     validateCerts: false
+  adminConnectors:
+    - type: http
+      port: 8081
+
+# Logging settings.
+logging:
+  # The default level of all loggers. Can be OFF, ERROR, WARN, INFO, DEBUG, TRACE, or ALL.
+  level: INFO
+  # Logger-specific levels.
+  loggers:
+    com.takipi: DEBUG
+
+  appenders:
+    - type: console

docker/scripts/run.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+java -jar /opt/takipi-storage/lib/takipi-storage.jar server /opt/takipi-storage/private/settings.yaml &> /opt/takipi-storage/log/takipi-storage.log &
+/usr/bin/tail -f /opt/takipi-storage/log/takipi-storage.log

pom.xml

@@ -11,7 +11,7 @@
 		<!-- use UTF-8 for everything -->
 		<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
 		<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
-		<dropwizard.version>1.2.2</dropwizard.version>
+		<dropwizard.version>0.7.1</dropwizard.version>
 		<dropwizard-debpkg-maven-plugin.version>0.7</dropwizard-debpkg-maven-plugin.version>
 	</properties>
 	<dependencies>
@@ -30,6 +30,16 @@
 			<artifactId>junit</artifactId>
 			<version>4.12</version>
 		</dependency>
+		<dependency>
+			<groupId>com.amazonaws</groupId>
+			<artifactId>aws-java-sdk-s3</artifactId>
+			<version>1.11.13</version>
+		</dependency>
+		<dependency>
+			<groupId>com.google.guava</groupId>
+			<artifactId>guava</artifactId>
+			<version>17.0</version>
+		</dependency>
 	</dependencies>
 	<build>
 		<plugins>
@@ -98,7 +108,7 @@
 		 <plugin>
 			 <groupId>org.codehaus.mojo</groupId>
 			 <artifactId>findbugs-maven-plugin</artifactId>
-			 <version>3.0.4-SNAPSHOT</version>
+			 <version>3.0.4</version>
 		 </plugin>
 	 </plugins>
  </reporting>

settings.yml

@@ -1,8 +1,31 @@
-folderPath: /opt/takipi-storage/storage
-maxUsedStoragePercentage: 0.95
 enableCors: true
 corsOrigins: "*"
 
+# If using attaching IAM Role to instance leave accessKey and secretKey empty
+s3Fs:
+    bucket: <bucket>
+    pathPrefix: <path_prefix>
+    credentials:
+      accessKey: <key> 
+      secretKey: <secret> 
+
+multifetch:
+
+    # maximum number of threads for concurrent multi-fetch. Set to zero to disable concurrent fetching.
+    concurrencyLevel: 100
+
+    # Recommended size >= 16777216. Set to zero to disable caching.
+    maxCacheSize: 67108864
+
+    enableCacheLogger: false
+
+    maxBatchSize: 4194304
+
+#folderFs:
+#  folderPath: /opt/takipi-storage/storage
+#  maxUsedStoragePercentage: 0.95
+
+
 server:
   gzip:
     includedMethods: [POST, GET]
@@ -11,9 +34,9 @@ server:
   applicationConnectors:
     - type: http
       port: 8080
-# this requires the alpn-boot library on the JVM's boot classpath
-#    - type: spdy3
-#      port: 8445
+# https support
+#    - type: https
+#      port: 8443
 #      keyStorePath: example.keystore
 #      keyStorePassword: example
 #      validateCerts: false