Merge pull request #350 from seejohnrun/specify-config-on-boot

Allow setting config.js alternative on boot

README.md

@@ -31,7 +31,7 @@ STDOUT.  Check the README there for more details and usages.
 1.  Download the package, and expand it
 2.  Explore the settings inside of config.js, but the defaults should be good
 3.  `npm install`
-4.  `npm start`
+4.  `npm start` (you may specify an optional `<config-path>` as well)
 
 ## Settings
 
@@ -52,7 +52,7 @@ STDOUT.  Check the README there for more details and usages.
 
 When present, the `rateLimits` option enables built-in rate limiting courtesy
 of `connect-ratelimit`.  Any of the options supported by that library can be
-used and set in `config.json`.
+used and set in `config.js`.
 
 See the README for [connect-ratelimit](https://github.com/dharmafly/connect-ratelimit)
 for more information!
@@ -154,6 +154,28 @@ or post.
 
 All of which are optional except `type` with very logical default values.
 
+### MongoDB
+
+To use mongodb storage you must install the 'mongodb' pachage in npm
+
+`npm install mongodb`
+
+Once you've done that, your config section should look like:
+
+``` json
+{
+  "type": "mongodb",
+  "connectionUrl": "mongodb://localhost:27017/database"
+}
+```
+
+You can also just set the environment variable for `DATABASE_URL` to your database connection url.
+
+Unlike with postgres you do NOT have to create the table in your mongo database prior to running.
+
+You can also set an `expire` option to the number of seconds to expire keys in.
+This is off by default, but will constantly kick back expirations on each view or post.
+
 ### Memcached
 
 To use memcache storage you must install the `memcached` package via npm
@@ -198,6 +220,22 @@ Also, you must create an `uploads` table, which will store all the data for uplo
 
 You can optionally add the `user` and `password` properties to use a user system.
 
+### Google Datastore
+
+To use the Google Datastore storage system, you must install the `@google-cloud/datastore` package via npm
+
+`npm install @google-cloud/datastore`
+
+Once you've done that, your config section should look like this:
+
+``` json
+{
+  "type": "google-datastore"
+}
+```
+
+Authentication is handled automatically by [Google Cloud service account credentials](https://cloud.google.com/docs/authentication/getting-started), by providing authentication details to the GOOGLE_APPLICATION_CREDENTIALS environmental variable.
+
 ### Amazon S3
 
 To use [Amazon S3](https://aws.amazon.com/s3/) as a storage system, you must
@@ -316,8 +354,6 @@ Here is a list of all the environment variables
 |  RATELIMITS_BLACKLIST_EVERY_SECONDS  |                                       |     By default client names in the blacklist will be subject to 0 requests per hours     |
 |         RATELIMITS_BLACKLIST         | example1.blacklist,example2.blacklist |           Comma separated list of the clients which are in the blacklistpool.            |
 
-
-
 ## Author
 
 John Crepezzi <john.crepezzi@gmail.com>

lib/document_handler.js

@@ -16,33 +16,55 @@ var DocumentHandler = function(options) {
 DocumentHandler.defaultKeyLength = 10;
 
 // Handle retrieving a document
-DocumentHandler.prototype.handleGet = function(key, response, skipExpire) {
+DocumentHandler.prototype.handleGet = function(request, response, config) {
+  const key = request.params.id.split('.')[0];
+  const skipExpire = !!config.documents[key];
+
   this.store.get(key, function(ret) {
     if (ret) {
       winston.verbose('retrieved document', { key: key });
       response.writeHead(200, { 'content-type': 'application/json' });
-      response.end(JSON.stringify({ data: ret, key: key }));
+      if (request.method === 'HEAD') {
+        response.end();
+      } else {
+        response.end(JSON.stringify({ data: ret, key: key }));
+      }
     }
     else {
       winston.warn('document not found', { key: key });
       response.writeHead(404, { 'content-type': 'application/json' });
-      response.end(JSON.stringify({ message: 'Document not found.' }));
+      if (request.method === 'HEAD') {
+        response.end();
+      } else {
+        response.end(JSON.stringify({ message: 'Document not found.' }));
+      }
     }
   }, skipExpire);
 };
 
 // Handle retrieving the raw version of a document
-DocumentHandler.prototype.handleRawGet = function(key, response, skipExpire) {
+DocumentHandler.prototype.handleRawGet = function(request, response, config) {
+  const key = request.params.id.split('.')[0];
+  const skipExpire = !!config.documents[key];
+
   this.store.get(key, function(ret) {
     if (ret) {
       winston.verbose('retrieved raw document', { key: key });
       response.writeHead(200, { 'content-type': 'text/plain; charset=UTF-8' });
-      response.end(ret);
+      if (request.method === 'HEAD') {
+        response.end();
+      } else {
+        response.end(ret);
+      }
     }
     else {
       winston.warn('raw document not found', { key: key });
       response.writeHead(404, { 'content-type': 'application/json' });
-      response.end(JSON.stringify({ message: 'Document not found.' }));
+      if (request.method === 'HEAD') {
+        response.end();
+      } else {
+        response.end(JSON.stringify({ message: 'Document not found.' }));
+      }
     }
   }, skipExpire);
 };

lib/document_stores/google-datastore.js

@@ -0,0 +1,89 @@
+/*global require,module,process*/
+
+const Datastore = require('@google-cloud/datastore');
+const winston = require('winston');
+
+class GoogleDatastoreDocumentStore {
+
+  // Create a new store with options
+  constructor(options) {
+    this.kind = "Haste";
+    this.expire = options.expire;
+    this.datastore = new Datastore();
+  }
+
+  // Save file in a key
+  set(key, data, callback, skipExpire) {
+    var expireTime = (skipExpire || this.expire === undefined) ? null : new Date(Date.now() + this.expire * 1000);
+
+    var taskKey = this.datastore.key([this.kind, key])
+    var task = {
+      key: taskKey,
+      data: [
+        {
+          name: 'value',
+          value: data,
+          excludeFromIndexes: true
+        },
+        {
+          name: 'expiration',
+          value: expireTime
+        }
+      ]
+    };
+
+    this.datastore.insert(task).then(() => {
+      callback(true);
+    })
+    .catch(err => {
+      callback(false);
+    });
+  }
+
+  // Get a file from a key
+  get(key, callback, skipExpire) {
+    var taskKey = this.datastore.key([this.kind, key])
+
+    this.datastore.get(taskKey).then((entity) => {
+      if (skipExpire || entity[0]["expiration"] == null) {
+        callback(entity[0]["value"]);
+      }
+      else {
+        // check for expiry
+        if (entity[0]["expiration"] < new Date()) {
+          winston.info("document expired", {key: key, expiration: entity[0]["expiration"], check: new Date(null)});
+          callback(false);
+        }
+        else {
+          // update expiry
+          var task = {
+            key: taskKey,
+            data: [
+              {
+                name: 'value',
+                value: entity[0]["value"],
+                excludeFromIndexes: true
+              },
+              {
+                name: 'expiration',
+                value: new Date(Date.now() + this.expire * 1000)
+              }
+            ]
+          };
+          this.datastore.update(task).then(() => {
+          })
+          .catch(err => {
+            winston.error("failed to update expiration", {error: err});
+          });
+          callback(entity[0]["value"]);
+        }
+      }
+    })
+    .catch(err => {
+        winston.error("Error retrieving value from Google Datastore", {error: err});
+        callback(false);
+    });
+  }
+}
+
+module.exports = GoogleDatastoreDocumentStore;

lib/document_stores/memcached.js

@@ -26,7 +26,7 @@ class MemcachedDocumentStore {
 
   // Save file in a key
   set(key, data, callback, skipExpire) {
-    this.client.set(key, data, skipExpire ? 0 : this.expire, (error) => {
+    this.client.set(key, data, skipExpire ? 0 : this.expire || 0, (error) => {
       callback(!error);
     });
   }
@@ -34,10 +34,12 @@ class MemcachedDocumentStore {
   // Get a file from a key
   get(key, callback, skipExpire) {
     this.client.get(key, (error, data) => {
-      callback(error ? false : data);
+      const value = error ? false : data;
+
+      callback(value);
 
       // Update the key so that the expiration is pushed forward
-      if (!skipExpire) {
+      if (value && !skipExpire) {
         this.set(key, data, (updateSucceeded) => {
           if (!updateSucceeded) {
             winston.error('failed to update expiration on GET', {key});

lib/document_stores/mongo.js

@@ -0,0 +1,88 @@
+
+
+var MongoClient = require('mongodb').MongoClient,
+    winston = require('winston');
+
+var MongoDocumentStore = function (options) {
+    this.expire = options.expire;
+    this.connectionUrl = process.env.DATABASE_URl || options.connectionUrl;
+};
+
+MongoDocumentStore.prototype.set = function (key, data, callback, skipExpire) {
+    var now = Math.floor(new Date().getTime() / 1000),
+        that = this;
+
+    this.safeConnect(function (err, db) {
+        if (err)
+            return callback(false);
+        
+        db.collection('entries').update({
+            'entry_id': key,
+            $or: [
+                { expiration: -1 },
+                { expiration: { $gt: now } }
+            ]
+        }, {
+            'entry_id': key,
+            'value': data,
+            'expiration': that.expire && !skipExpire ? that.expire + now : -1
+        }, {
+            upsert: true
+        }, function (err, existing) {
+            if (err) {
+                winston.error('error persisting value to mongodb', { error: err });
+                return callback(false);
+            }
+
+            callback(true);
+        });
+    });
+};
+
+MongoDocumentStore.prototype.get = function (key, callback, skipExpire) {
+    var now = Math.floor(new Date().getTime() / 1000),
+        that = this;
+
+    this.safeConnect(function (err, db) {
+        if (err)
+            return callback(false);
+        
+        db.collection('entries').findOne({
+            'entry_id': key,
+            $or: [
+                { expiration: -1 },
+                { expiration: { $gt: now } }
+            ]
+        }, function (err, entry) {
+            if (err) {
+                winston.error('error persisting value to mongodb', { error: err });
+                return callback(false);
+            }
+
+            callback(entry === null ? false : entry.value);
+
+            if (entry !== null && entry.expiration !== -1 && that.expire && !skipExpire) {
+                db.collection('entries').update({
+                    'entry_id': key
+                }, {
+                    $set: {
+                        'expiration': that.expire + now
+                    }
+                }, function (err, result) { });
+            }
+        });
+    });
+};
+
+MongoDocumentStore.prototype.safeConnect = function (callback) {
+    MongoClient.connect(this.connectionUrl, function (err, db) {
+        if (err) {
+            winston.error('error connecting to mongodb', { error: err });
+            callback(err);
+        } else {
+            callback(undefined, db);
+        }
+    });
+};
+
+module.exports = MongoDocumentStore;

lib/document_stores/postgres.js

@@ -1,14 +1,16 @@
 /*global require,module,process*/
 
-var postgres = require('pg');
 var winston = require('winston');
+const {Pool} = require('pg');
 
 // create table entries (id serial primary key, key varchar(255) not null, value text not null, expiration int, unique(key));
 
 // A postgres document store
 var PostgresDocumentStore = function (options) {
   this.expireJS = options.expire;
-  this.connectionUrl = process.env.DATABASE_URL || options.connectionUrl;
+
+  const connectionString = process.env.DATABASE_URL || options.connectionUrl;
+  this.pool = new Pool({connectionString});
 };
 
 PostgresDocumentStore.prototype = {
@@ -64,16 +66,15 @@ PostgresDocumentStore.prototype = {
 
   // A connection wrapper
   safeConnect: function (callback) {
-    postgres.connect(this.connectionUrl, function (err, client, done) {
-      if (err) {
-        winston.error('error connecting to postgres', { error: err });
-        callback(err);
+    this.pool.connect((error, client, done) => {
+      if (error) {
+        winston.error('error connecting to postgres', {error});
+        callback(error);
       } else {
         callback(undefined, client, done);
       }
     });
   }
-
 };
 
 module.exports = PostgresDocumentStore;

package-lock.json

@@ -156,9 +156,9 @@
       }
     },
     "buffer-writer": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/buffer-writer/-/buffer-writer-1.0.1.tgz",
-      "integrity": "sha1-Iqk2kB4wKa/NdUfrRIfOtpejvwg="
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/buffer-writer/-/buffer-writer-2.0.0.tgz",
+      "integrity": "sha512-a7ZpuTZU1TRtnwyCNW3I5dc0wWNC3VR9S++Ewyk2HHZdrO3CQJqSpd+95Us590V6AL7JqUAH2IwZ/398PmNFgw=="
     },
     "busboy": {
       "version": "0.2.4",
@@ -547,11 +547,6 @@
       "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz",
       "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A=="
     },
-    "generic-pool": {
-      "version": "2.4.3",
-      "resolved": "https://registry.npmjs.org/generic-pool/-/generic-pool-2.4.3.tgz",
-      "integrity": "sha1-eAw29p360FpaBF3Te+etyhGk9v8="
-    },
     "get-caller-file": {
       "version": "2.0.5",
       "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz",
@@ -773,11 +768,6 @@
         "iterate-iterator": "^1.0.1"
       }
     },
-    "js-string-escape": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/js-string-escape/-/js-string-escape-1.0.1.tgz",
-      "integrity": "sha1-4mJbrbwNZ8dTPp7cEGjFh65BN+8="
-    },
     "js-yaml": {
       "version": "3.14.0",
       "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.0.tgz",
@@ -909,11 +899,6 @@
       "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==",
       "dev": true
     },
-    "object-assign": {
-      "version": "4.1.0",
-      "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.0.tgz",
-      "integrity": "sha1-ejs9DpgGPUP0wD8uiubNUahog6A="
-    },
     "object-inspect": {
       "version": "1.8.0",
       "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.8.0.tgz",
@@ -978,9 +963,9 @@
       "dev": true
     },
     "packet-reader": {
-      "version": "0.3.1",
-      "resolved": "https://registry.npmjs.org/packet-reader/-/packet-reader-0.3.1.tgz",
-      "integrity": "sha1-zWLmCvjX/qinBexP+ZCHHEaHHyc="
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/packet-reader/-/packet-reader-1.0.0.tgz",
+      "integrity": "sha512-HAKu/fG3HpHFO0AA8WE8q2g+gBJaZ9MG7fcKk+IJPLTGAD6Psw4443l+9DGRbOIh3/aXr7Phy0TjilYivJo5XQ=="
     },
     "parseurl": {
       "version": "1.3.3",
@@ -1000,24 +985,23 @@
       "dev": true
     },
     "pg": {
-      "version": "6.4.2",
-      "resolved": "https://registry.npmjs.org/pg/-/pg-6.4.2.tgz",
-      "integrity": "sha1-w2QBEGDqx6UHoq4GPrhX7OkQ4n8=",
+      "version": "8.4.0",
+      "resolved": "https://registry.npmjs.org/pg/-/pg-8.4.0.tgz",
+      "integrity": "sha512-01LcNrAf+mBI46c78mE86I5o5KkOM942lLiSBdiCfgHTR+oUNIjh1fKClWeoPNHJz2oXe/VUSqtk1vwAQYwWEg==",
       "requires": {
-        "buffer-writer": "1.0.1",
-        "js-string-escape": "1.0.1",
-        "packet-reader": "0.3.1",
-        "pg-connection-string": "0.1.3",
-        "pg-pool": "1.*",
-        "pg-types": "1.*",
-        "pgpass": "1.*",
-        "semver": "4.3.2"
+        "buffer-writer": "2.0.0",
+        "packet-reader": "1.0.0",
+        "pg-connection-string": "^2.4.0",
+        "pg-pool": "^3.2.1",
+        "pg-protocol": "^1.3.0",
+        "pg-types": "^2.1.0",
+        "pgpass": "1.x"
       }
     },
     "pg-connection-string": {
-      "version": "0.1.3",
-      "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-0.1.3.tgz",
-      "integrity": "sha1-2hhHsglA5C7hSSvq9l1J2RskXfc="
+      "version": "2.4.0",
+      "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.4.0.tgz",
+      "integrity": "sha512-3iBXuv7XKvxeMrIgym7njT+HlZkwZqqGX4Bu9cci8xHZNT+Um1gWKqCsAzcC0d95rcKMU5WBg6YRUcHyV0HZKQ=="
     },
     "pg-int8": {
       "version": "1.0.1",
@@ -1025,23 +1009,24 @@
       "integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw=="
     },
     "pg-pool": {
-      "version": "1.8.0",
-      "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-1.8.0.tgz",
-      "integrity": "sha1-9+xzgkw3oD8Hb1G/33DjQBR8Tzc=",
-      "requires": {
-        "generic-pool": "2.4.3",
-        "object-assign": "4.1.0"
-      }
+      "version": "3.2.1",
+      "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.2.1.tgz",
+      "integrity": "sha512-BQDPWUeKenVrMMDN9opfns/kZo4lxmSWhIqo+cSAF7+lfi9ZclQbr9vfnlNaPr8wYF3UYjm5X0yPAhbcgqNOdA=="
+    },
+    "pg-protocol": {
+      "version": "1.3.0",
+      "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.3.0.tgz",
+      "integrity": "sha512-64/bYByMrhWULUaCd+6/72c9PMWhiVFs3EVxl9Ct6a3v/U8+rKgqP2w+kKg/BIGgMJyB+Bk/eNivT32Al+Jghw=="
     },
     "pg-types": {
-      "version": "1.13.0",
-      "resolved": "https://registry.npmjs.org/pg-types/-/pg-types-1.13.0.tgz",
-      "integrity": "sha512-lfKli0Gkl/+za/+b6lzENajczwZHc7D5kiUCZfgm914jipD2kIOIvEkAhZ8GrW3/TUoP9w8FHjwpPObBye5KQQ==",
+      "version": "2.2.0",
+      "resolved": "https://registry.npmjs.org/pg-types/-/pg-types-2.2.0.tgz",
+      "integrity": "sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==",
       "requires": {
         "pg-int8": "1.0.1",
-        "postgres-array": "~1.0.0",
+        "postgres-array": "~2.0.0",
         "postgres-bytea": "~1.0.0",
-        "postgres-date": "~1.0.0",
+        "postgres-date": "~1.0.4",
         "postgres-interval": "^1.1.0"
       }
     },
@@ -1060,9 +1045,9 @@
       "dev": true
     },
     "postgres-array": {
-      "version": "1.0.3",
-      "resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-1.0.3.tgz",
-      "integrity": "sha512-5wClXrAP0+78mcsNX3/ithQ5exKvCyK5lr5NEEEeGwwM6NJdQgzIJBVxLvRW+huFpX92F2QnZ5CcokH0VhK2qQ=="
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-2.0.0.tgz",
+      "integrity": "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA=="
     },
     "postgres-bytea": {
       "version": "1.0.0",
@@ -1159,11 +1144,6 @@
       "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz",
       "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ=="
     },
-    "semver": {
-      "version": "4.3.2",
-      "resolved": "https://registry.npmjs.org/semver/-/semver-4.3.2.tgz",
-      "integrity": "sha1-x6BxWKgL7dBSNVt3DYLWZA+AO+c="
-    },
     "serialize-javascript": {
       "version": "4.0.0",
       "resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-4.0.0.tgz",

package.json

@@ -18,7 +18,7 @@
     "connect": "^3.7.0",
     "connect-ratelimit": "0.0.7",
     "connect-route": "0.1.5",
-    "pg": "^6.0.0",
+    "pg": "^8.0.0",
     "redis": "0.8.1",
     "redis-url": "0.1.0",
     "st": "^2.0.0",

server.js

@@ -11,7 +11,8 @@ var connect_rate_limit = require('connect-ratelimit');
 var DocumentHandler = require('./lib/document_handler');
 
 // Load the configuration and set some defaults
-var config = JSON.parse(fs.readFileSync('./config.js', 'utf8'));
+const configPath = process.argv.length <= 2 ? 'config.js' : process.argv[2];
+const config = JSON.parse(fs.readFileSync(configPath, 'utf8'));
 config.port = process.env.PORT || config.port || 7777;
 config.host = process.env.HOST || config.host || 'localhost';
 
@@ -109,20 +110,28 @@ if (config.rateLimits) {
 // first look at API calls
 app.use(route(function(router) {
   // get raw documents - support getting with extension
+
   router.get('/raw/:id', function(request, response) {
-    var key = request.params.id.split('.')[0];
-    var skipExpire = !!config.documents[key];
-    return documentHandler.handleRawGet(key, response, skipExpire);
+    return documentHandler.handleRawGet(request, response, config);
   });
+
+  router.head('/raw/:id', function(request, response) {
+    return documentHandler.handleRawGet(request, response, config);
+  });
+
   // add documents
+
   router.post('/documents', function(request, response) {
     return documentHandler.handlePost(request, response);
   });
+
   // get documents
   router.get('/documents/:id', function(request, response) {
-    var key = request.params.id.split('.')[0];
-    var skipExpire = !!config.documents[key];
-    return documentHandler.handleGet(key, response, skipExpire);
+    return documentHandler.handleGet(request, response, config);
+  });
+
+  router.head('/documents/:id', function(request, response) {
+    return documentHandler.handleGet(request, response, config);
   });
 }));
 

static/application.css

@@ -33,6 +33,7 @@ textarea {
 	font-size: 13px;
 	font-family: monospace;
 	text-align: right;
+	user-select: none;
 }
 
 /* code box when locked */
@@ -119,6 +120,7 @@ textarea {
 	font-size: 12px;
 	line-height: 14px;
 	padding: 10px 15px;
+	user-select: none;
 }
 
 #box3 .label, #messages li {

static/application.js

@@ -64,7 +64,7 @@ haste_document.prototype.save = function(data, callback) {
     type: 'post',
     data: data,
     dataType: 'json',
-    contentType: 'application/json; charset=utf-8',
+    contentType: 'text/plain; charset=utf-8',
     success: function(res) {
       _this.locked = true;
       _this.key = res.key;