The legacy distributed object storage server developed by PitchPoint Solutions can store billions of large and small files using minimal resources. Object data is stored in replicated volumes implemented like Facebooks Haystack Object Store. Object metadata which essentially maps an object name to a volume position is stored in an elasticsearch index. (Development by PitchPoint Solutions has been discontinued)
APACHE-2.0 License
The most recent release of sfs is release-1.20170106133707.
To run release-1.20170106133707
docker run ... pitchpointsolutions/simple-file-server:release-1.20170106133707
To run the latest release-1
docker pull pitchpointsolutions/simple-file-server:release-1
docker run ... pitchpointsolutions/simple-file-server:release-1
Snapshots of sfs are built from the master branch
To run the latest snapshot
docker pull pitchpointsolutions/simple-file-server:latest
docker run ... pitchpointsolutions/simple-file-server:latest
docker run -d -P --name sfs_example_elasticsearch elasticsearch:2.4.1 -Des.cluster.name=sfs_example_elasticsearch
HOSTNAME=`hostname` && export HOST_IP=`ping -c1 -n ${HOSTNAME} | head -n1 | sed "s/.*(\([0-9]*\.[0-9]*\.[0-9]*\.[0-9]*\)).*/\1/g"`;
export DOCKER_ES_PORT=`docker port sfs_example_elasticsearch 9300/tcp | sed -E 's/(.+):(.+)/\2/'`
docker run -d -P --add-host localhost:127.0.0.1 -e SFS_HTTP_LISTEN_ADDRESSES=0.0.0.0:80 -e SFS_HTTP_PUBLISH_ADDRESSES=127.0.0.1:80 -e SFS_REMOTENODE_SECRET=YWJjMTIzCg== -e SFS_KEYSTORE_AWS_KMS_ENDPOINT=https://kms.us-east-1.amazonaws.com -e SFS_KEYSTORE_AWS_KMS_KEY_ID=arn:aws:kms:us-east-1:111122223333:key/1234abcd-12ab-34cd-56ef-1234567890ab -e SFS_KEYSTORE_AWS_KMS_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE -e SFS_KEYSTORE_AWS_KMS_SECRET_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYAWSEXAMPLEKEY -e SFS_KEYSTORE_AZURE_KMS_ENDPOINT=https://yourvaultname.vault.azure.net -e SFS_KEYSTORE_AZURE_KMS_KEY_ID=6603bbb5-cf2e-4367-8327-43ba49ba74b0 -e SFS_KEYSTORE_AZURE_KMS_ACCESS_KEY_ID=a14970c2-397c-4af2-867e-b3480f9eaac6 -e SFS_KEYSTORE_AZURE_KMS_SECRET_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYAZUREEXAMPLEKEY -e SFS_ELASTICSEARCH_CLUSTER_NAME=sfs_example_elasticsearch -e SFS_ELASTICSEARCH_NODE_NAME=${HOST_IP}:${DOCKER_ES_PORT} -e SFS_ELASTICSEARCH_DISCOVERY_ZEN_PING_UNICAST_HOSTS=${HOST_IP}:${DOCKER_ES_PORT} -e SFS_ELASTICSEARCH_DISCOVERY_ZEN_PING_MULTICAST_ENABLED=false -e SFS_ELASTICSEARCH_DISCOVERY_ZEN_PING_UNICAST_ENABLED=true --detach --name sfs_example_middlware -P pitchpointsolutions/simple-file-server
export DOCKER_SFS_PORT=`docker port sfs_example_middlware 80/tcp | sed -E 's/(.+):(.+)/\2/'`
curl -v -XGET "http://localhost:${DOCKER_SFS_PORT}/admin/001/healthcheck"
curl -XPOST -u admin:admin "http://localhost:${DOCKER_SFS_PORT}/openstackswift001/my_account"
curl -XPUT -u admin:admin "http://localhost:${DOCKER_SFS_PORT}/openstackswift001/my_account/my-container"
curl -XPUT -u admin:admin "http://localhost:${DOCKER_SFS_PORT}/openstackswift001/my_account/my-container/my_object" -d 'abc123'
curl -XGET -u admin:admin "http://localhost:${DOCKER_SFS_PORT}/openstackswift001/my_account/my-container/my_object"
docker run -d -P --name sfs_example_elasticsearch elasticsearch:2.4.1 -Des.cluster.name=sfs_example_elasticsearch
export HOST_IP=`docker-machine ip`;
export DOCKER_ES_PORT=`docker port sfs_example_elasticsearch 9300/tcp | sed -E 's/(.+):(.+)/\2/'`
docker run -d -P --add-host localhost:127.0.0.1 -e SFS_HTTP_LISTEN_ADDRESSES=0.0.0.0:80 -e SFS_HTTP_PUBLISH_ADDRESSES=127.0.0.1:80 -e SFS_REMOTENODE_SECRET=YWJjMTIzCg== -e SFS_KEYSTORE_AWS_KMS_ENDPOINT=https://kms.us-east-1.amazonaws.com -e SFS_KEYSTORE_AWS_KMS_KEY_ID=arn:aws:kms:us-east-1:111122223333:key/1234abcd-12ab-34cd-56ef-1234567890ab -e SFS_KEYSTORE_AWS_KMS_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE -e SFS_KEYSTORE_AWS_KMS_SECRET_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYAWSEXAMPLEKEY -e SFS_KEYSTORE_AZURE_KMS_ENDPOINT=https://yourvaultname.vault.azure.net -e SFS_KEYSTORE_AZURE_KMS_KEY_ID=6603bbb5-cf2e-4367-8327-43ba49ba74b0 -e SFS_KEYSTORE_AZURE_KMS_ACCESS_KEY_ID=a14970c2-397c-4af2-867e-b3480f9eaac6 -e SFS_KEYSTORE_AZURE_KMS_SECRET_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYAZUREEXAMPLEKEY -e SFS_ELASTICSEARCH_CLUSTER_NAME=sfs_example_elasticsearch -e SFS_ELASTICSEARCH_NODE_NAME=${HOST_IP}:${DOCKER_ES_PORT} -e SFS_ELASTICSEARCH_DISCOVERY_ZEN_PING_UNICAST_HOSTS=${HOST_IP}:${DOCKER_ES_PORT} -e SFS_ELASTICSEARCH_DISCOVERY_ZEN_PING_MULTICAST_ENABLED=false -e SFS_ELASTICSEARCH_DISCOVERY_ZEN_PING_UNICAST_ENABLED=true --detach --name sfs_example_middlware -P pitchpointsolutions/simple-file-server
export DOCKER_SFS_PORT=`docker port sfs_example_middlware 80/tcp | sed -E 's/(.+):(.+)/\2/'`
curl -v -XGET "http://${HOST_IP}:${DOCKER_SFS_PORT}/admin/001/healthcheck"
curl -XPOST -u admin:admin "http://${HOST_IP}:${DOCKER_SFS_PORT}/openstackswift001/my_account"
curl -XPUT -u admin:admin "http://${HOST_IP}:${DOCKER_SFS_PORT}/openstackswift001/my_account/my-container"
curl -XPUT -u admin:admin "http://${HOST_IP}:${DOCKER_SFS_PORT}/openstackswift001/my_account/my-container/my_object" -d 'abc123'
curl -XGET -u admin:admin "http://${HOST_IP}:${DOCKER_SFS_PORT}/openstackswift001/my_account/my-container/my_object"
Oracle Java 8 and Maven 3 should be used for building.
SFS_KEYSTORE_AWS_KMS_ENDPOINT=https://kms.us-east-1.amazonaws.com
SFS_KEYSTORE_AWS_KMS_KEY_ID=...
SFS_KEYSTORE_AWS_KMS_ACCESS_KEY_ID=$...
SFS_KEYSTORE_AWS_KMS_SECRET_KEY=...
SFS_KEYSTORE_AZURE_KMS_ENDPOINT=https://....vault.azure.net
SFS_KEYSTORE_AZURE_KMS_KEY_ID=...
SFS_KEYSTORE_AZURE_KMS_ACCESS_KEY_ID=...
SFS_KEYSTORE_AZURE_KMS_SECRET_KEY=...
mvn clean package
mvn clean package
mvn clean package -Pprotoc
docker run -it --add-host "es-host:${HOST_IP}" --add-host "localhost:127.0.0.1" -e "INSTANCES=200" -e "HEAP_SIZE=512m" -p 8092:80 -v ${PWD}:/data -v ${PWD}/sample-config.json:/etc/vertx-conf.json -v ${PWD}/sample-logback.xml:/etc/vertx-logback.xml pps-sfs:latest
{
"auth": {
"admin": [
{
"id": 1,
"password": "admin",
"username": "admin"
}
],
"user": [
{
"id": 2,
"password": "user",
"username": "user"
}
]
},
"elasticsearch.cluster.name": "elasticsearch_samplecluster",
"elasticsearch.defaultdeletetimeout": 30000,
"elasticsearch.defaultgettimeout": 30000,
"elasticsearch.defaultindextimeout": 30000,
"elasticsearch.defaultsearchtimeout": 30000,
"elasticsearch.discovery.zen.ping.multicast.enabled": false,
"elasticsearch.discovery.zen.ping.unicast.enabled": true,
"elasticsearch.discovery.zen.ping.unicast.hosts": [
"es-host:9300"
],
"elasticsearch.node.name": "simple-file-server-client",
"elasticsearch.replicas": 0,
"elasticsearch.shards": 1,
"fs.home": "/data",
"http.listen.addresses": [
"0.0.0.0:80"
],
"http.maxheadersize": 40960,
"http.publish.addresses": [
"${docker_image_ip_address}:8092"
],
"keystore.aws.kms.access_key_id": "${aws_kms_access_key_id}",
"keystore.aws.kms.endpoint": "https://kms.us-east-1.amazonaws.com",
"keystore.aws.kms.key_id": "${aws_kms_key_id}",
"keystore.aws.kms.secret_key": "${aws_kms_access_key_id}",
"keystore.azure.kms.access_key_id": "${aws_azure_access_key_id}",
"keystore.azure.kms.endpoint": "https://....vault.azure.net",
"keystore.azure.kms.key_id": "${aws_azure_key_id}",
"keystore.azure.kms.secret_key": "${aws_azure_secret_key}",
"node.data": true,
"node.master": true,
"number_of_object_replicas": 1,
"remotenode.connectimeout": 5000,
"remotenode.maxpoolsize": 200,
"remotenode.responsetimeout": 10000,
"remotenode.secret": "YWJjMTIzCg==",
"threadpool.background.size": 200,
"threadpool.io.size": 200
}
<?xml version="1.0" encoding="UTF-8" ?>
<configuration scan="true">
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
<!-- encoders are assigned by default the type
ch.qos.logback.classic.encoder.PatternLayoutEncoder -->
<encoder>
<pattern>%d [%thread] %-5level %logger{36} - %msg%n</pattern>
</encoder>
</appender>
<logger name="org.apache" level="TRACE" additivity="false">
<appender-ref ref="STDOUT"/>
</logger> d
<logger name="io.vertx" level="TRACE" additivity="false">
<appender-ref ref="STDOUT"/>
</logger>
<logger name="io.netty" level="DEBUG" additivity="false">
<appender-ref ref="STDOUT"/>
</logger>
<logger name="org.sfs" level="INFO" additivity="false">
<appender-ref ref="STDOUT"/>
</logger>
</configuration>
curl -XPOST -u admin:admin "http://localhost:8092/openstackswift001/my_account"
curl -XHEAD -u admin:admin "http://localhost:8092/openstackswift001/my_account"
curl -XGET -u admin:admin "http://localhost:8092/openstackswift001/my_account"
curl -XDELETE -u admin:admin "http://localhost:8092/openstackswift001/my_account"
curl -XPUT -u admin:admin "http://localhost:8092/openstackswift001/my_account/my-container"
curl -XPOST -u admin:admin -H "X-Container-Meta-Server-Side-Encryption: true" -H "X-Container-Meta-Max-Object-Revisions: 3" "http://localhost:8092/openstackswift001/my_account/my-container"
curl -XHEAD -u admin:admin "http://localhost:8092/openstackswift001/my_account/my-container"
curl -XGET -u admin:admin "http://localhost:8092/openstackswift001/my_account/my-container?format=xml&prefix=&limit=10000&delimiter=%2F"
curl -XGET -u admin:admin "http://localhost:8092/sfs/openstackswift001/my_account/my-container?format=xml&prefix=subfolder%2F&limit=10000&delimiter=%2F
curl -XPUT -u admin:admin -H "X-Container-Meta-Server-Side-Encryption: true" -H "X-Container-Meta-Max-Object-Revisions: 3" "http://localhost:8092/openstackswift001/my_account/my-container"
curl -XPUT -u admin:admin -H "x-sfs-object-index-shards: 12" -H "x-sfs-object-index-replicas: 2" -H "x-sfs-object-replicas: 2" "http://localhost:8092/openstackswift001/my_account/my-container"
curl -XPOST -u admin:admin -H "X-Container-Meta-Server-Side-Encryption: true" -H "X-Container-Meta-Max-Object-Revisions: 2" "http://localhost:8092/openstackswift001/my_account/my-container"
curl -XPOST -u admin:admin -H "X-Container-Meta-Server-Side-Encryption: false" -H "X-Container-Meta-Max-Object-Revisions: 1" "http://localhost:8092/openstackswift001/my_account/my-container"
curl -XPOST -u admin:admin -H "x-sfs-object-index-replicas: 1" -H "x-sfs-object-replicas: 1" "http://localhost:8092/openstackswift001/my_account/my-container"
curl -XDELETE -u admin:admin "http://localhost:8092/openstackswift001/my_account/my-container"
curl -XPUT -u admin:admin "http://localhost:8092/openstackswift001/my_account/my-container/my_object" -d 'abc123'
curl -XPUT -u admin:admin -H "X-Server-Side-Encryption: true" "http://localhost:8092/openstackswift001/my_account/my-container/my_object" -d 'abc123'
curl -XHEAD -u admin:admin "http://localhost:8092/openstackswift001/my_account/my-container/my_object"
curl -XGET -u admin:admin "http://localhost:8092/openstackswift001/my_account/my-container/my_object"
curl -XDELETE -u admin:admin "http://localhost:8092/openstackswift001/my_account/my-container/my_object"
curl -XDELETE -u admin:admin "http://localhost:8092/openstackswift001/my_account/my-container/my_object?version=all"
curl -XDELETE -u admin:admin "http://localhost:8092/openstackswift001/my_account/my-container/my_object?version=1"
curl -XDELETE -u admin:admin "http://localhost:8092/openstackswift001/my_account/my-container/my_object?version=1,2,3"
curl -XPOST -u admin:admin -H "x-sfs-dest-directory: /data/my-container_export" "http://localhost:8092/export_container/my_account/my-container"
curl -XPOST -u admin:admin -H "x-sfs-dest-directory: /data/my-container_export" -H "x-sfs-compress: true" "http://localhost:8092/export_container/my_account/my-container"
curl -XPOST -u admin:admin -H "x-sfs-dest-directory: /data/my-container_export" -H "x-sfs-compress: true" -H "x-sfs-secret: YWJjMTIzCg==" "http://localhost:8092/export_container/my_account/my-container"
curl -XPOST -u admin:admin -H "x-sfs-dest-directory: /data/my-container_export" "http://localhost:8092/import_container/my_account/target_containers"
curl -XPOST -u admin:admin -H "x-sfs-src-directory: /data/my-container_export" -H "x-sfs-secret: YWJjMTIzCg==" "http://localhost:8092/import_container/my_account/target_container"
curl -XGET -u admin:admin "http://localhost:8092/admin/001/healthcheck"
curl -XPOST -u admin:admin "http://localhost:8092/verify_repair_objects/my_account/my-container/my_object"
curl -XPOST -u admin:admin "http://localhost:8092/verify_repair_objects/my_account/my-container"
curl -XGET -u admin:admin -H "timeout: 30000" "http://localhost:8092/verify_repair_objects/my_account/my-container"
curl -XDELETE -u admin:admin -H "timeout: 30000" "http://localhost:8092/verify_repair_objects/my_account/my-container"
curl -XPOST -u admin:admin "http://localhost:8092/verify_repair_objects"
curl -XGET -u admin:admin -H "timeout: 30000" "http://localhost:8092/verify_repair_objects"
curl -XDELTE -u admin:admin -H "timeout: 30000" "http://localhost:8092/verify_repair_objects"
curl -XPOST -u admin:admin "http://localhost:8092/verify_repair_masterkeys"
curl -XGET -u admin:admin -H "timeout: 30000" "http://localhost:8092/verify_repair_masterkeys"
curl -XPOST -u admin:admin "http://localhost:8092/reencrypt_containerkeys"
curl -XGET -u admin:admin -H "timeout: 30000" "http://localhost:8092/reencrypt_containerkeys"
curl -XPOST -u admin:admin "http://localhost:8092/reencrypt_masterkeys"
curl -XGET -u admin:admin -H "timeout: 30000" "http://localhost:8092/reencrypt_masterkeys"
curl -XGET -u admin:admin -H "timeout: 30000" "http://localhost:8092/metadata_objects/my_account/my-container/my_object"
curl -XGET -u admin:admin -H "timeout: 30000" "http://localhost:8092/metadata_containers/my_account/my-container"
curl -XGET -u admin:admin -H "timeout: 30000" "http://localhost:8092/metadata_accounts/my_account"