Added support for modular, composable presentation markdown.

app/Module.java

@@ -51,6 +51,8 @@ public class Module extends AbstractModule {
         bind(ImageService.class).asEagerSingleton();
         bind(VideoService.class).asEagerSingleton();
         bind(GISTService.class).asEagerSingleton();
+        bind(WebService.class).asEagerSingleton();
+        bind(ComposableService.class).asEagerSingleton();
         bind(GRSManager.class).asEagerSingleton();
         bind(GitHub.class).asEagerSingleton();
         bind(GitLab.class).asEagerSingleton();

app/com/gitpitch/services/ComposableService.java

@@ -0,0 +1,227 @@
+/*
+ * MIT License
+ *
+ * Copyright (c) 2016 David Russell
+ * 
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ * 
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ * 
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+package com.gitpitch.services;
+
+import com.gitpitch.utils.PitchParams;
+import com.gitpitch.git.GRS;
+import com.gitpitch.git.GRSService;
+import com.gitpitch.services.WebService;
+import com.gitpitch.models.MarkdownModel;
+import play.Logger;
+
+import javax.inject.*;
+import java.net.HttpURLConnection;
+import java.nio.charset.StandardCharsets;
+import java.util.Map;
+import java.util.Arrays;
+import java.util.List;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+/*
+ * Composable service supporting Markdown includes in
+ * the form of:
+ * 
+ * ---?include=/path/to/some.md
+ * +++?include=/path/to/other.md
+ */
+@Singleton
+public class ComposableService {
+
+    private final Logger.ALogger log = Logger.of(this.getClass());
+
+    private final DiskService ds;
+    private final WebService ws;
+
+    @Inject
+    public ComposableService(DiskService ds,
+                                     WebService ws) {
+
+        this.ds = ds;
+        this.ws = ws;
+    }
+
+    /*
+     * Make best effort to process composable presentation
+     * includes within PITCHME.md.
+     */
+    public void compose(PitchParams pp,
+                        GRS grs,
+                        GRSService grsService) {
+
+        try {
+
+            Path mdPath = ds.asPath(pp, PitchParams.PITCHME_MD);
+
+            String composed = null;
+            try (Stream<String> stream = Files.lines(mdPath)) {
+
+                composed = stream.map(md -> {
+
+                    if(md.startsWith(HSLIDE_INCLUDE)) {
+
+                        String included = handleInclude(pp,
+                                                        md,
+                                                        HSLIDE_INCLUDE,
+                                                        grsService,
+                                                        grs.getHeaders());
+                        if(included != null) {
+                            return MarkdownModel.HSLIDE_DELIM_DEFAULT +
+                                    NEWLINES + included;
+                        } else {
+                            return notFound(MarkdownModel.HSLIDE_DELIM_DEFAULT,
+                                    includePath(md, HSLIDE_INCLUDE));
+                        }
+                    } else
+                    if(md.startsWith(VSLIDE_INCLUDE)) {
+
+                        String included = handleInclude(pp,
+                                                        md,
+                                                        VSLIDE_INCLUDE,
+                                                        grsService,
+                                                        grs.getHeaders());
+                        if(included != null) {
+                            return MarkdownModel.VSLIDE_DELIM_DEFAULT +
+                                    NEWLINES + included;
+                        } else {
+                            return notFound(MarkdownModel.VSLIDE_DELIM_DEFAULT,
+                                    includePath(md, VSLIDE_INCLUDE));
+                        }
+                    } else {
+                        return md;
+                    }
+                }).collect(Collectors.joining("\n"));
+
+                boolean overwritten = overwrite(pp, mdPath, composed);
+                log.debug("compose: overwritten md={}", overwritten);
+
+            } catch (Exception mex) {
+                log.warn("Markdown processing ex={}", mex);
+                composed = "PITCHME.md could not be parsed.";
+            }
+
+        } catch(Exception stex) {
+
+        }
+    }
+
+    private String handleInclude(PitchParams pp,
+                                 String md,
+                                 String includeDelim,
+                                 GRSService grsService,
+                                 Map<String,String> headers) {
+
+        String included = null;
+        String includePath = includePath(md, includeDelim);
+
+        if(FORBIDDEN.contains(includePath)) {
+            included = notPermitted(includeDelim, includePath);
+        } else {
+            included = fetchInclude(pp, includePath,
+                    includeDelim, grsService, headers);
+        }
+        return included;
+    }
+
+    private String fetchInclude(PitchParams pp,
+                                String includePath,
+                                String includeDelim,
+                                GRSService grsService,
+                                Map<String,String> headers) {
+
+        String fetched = null;
+
+        try {
+
+            if(!isAbsolute(includePath))
+                includePath = grsService.raw(pp, includePath, true);
+
+            fetched = ws.fetchText(pp, includePath, headers);
+
+        } catch(Exception fiex) {
+            log.warn("fetchInclude: pp={}, error={}", pp, fiex);
+        }
+
+        return fetched;
+    }
+
+    private String includePath(String md, String includeDelim) {
+        try {
+            return md.substring(includeDelim.length());
+        } catch(Exception ipex) {
+            return md;
+        }
+    }
+
+    private boolean overwrite(PitchParams pp,
+                              Path mdPath,
+                              String stitched) {
+        try {
+            Files.write(mdPath, stitched.getBytes(),
+                    StandardOpenOption.CREATE,
+                    StandardOpenOption.TRUNCATE_EXISTING );
+            return true;
+        } catch(Exception fex) {
+            log.warn("overwrite: pp={}, write error={}", pp, fex);
+            return false;
+        }
+    }
+
+    private String notFound(String delim, String path) {
+        StringBuffer buf = new StringBuffer(delim);
+        buf.append(NEWLINES)
+           .append("### GitPitch ?Include")
+           .append(NEWLINES).append(".").append(NEWLINES)
+           .append(path)
+           .append(NEWLINES).append(".").append(NEWLINES)
+           .append("Markdown File Not Found");
+        return buf.toString();
+    }
+
+    private String notPermitted(String delim, String path) {
+        StringBuffer buf = new StringBuffer(NEWLINES);
+        buf.append("### GitPitch ?Include")
+           .append(NEWLINES).append(".").append(NEWLINES)
+           .append(path)
+           .append(NEWLINES).append(".").append(NEWLINES)
+           .append("Path Not Permitted");
+        return buf.toString();
+    }
+
+    private boolean isAbsolute(String path) {
+        return path.startsWith(HTTP);
+    }
+
+    public final String HSLIDE_INCLUDE =
+        MarkdownModel.HSLIDE_DELIM_DEFAULT + "?include=";
+    public final String VSLIDE_INCLUDE =
+        MarkdownModel.VSLIDE_DELIM_DEFAULT + "?include=";
+    private final String HTTP = "http";
+    private final String NEWLINES = "\n\n";
+    private final List<String> FORBIDDEN =
+        Arrays.asList(PitchParams.PITCHME_MD, "./PITCHME.md");
+}

app/com/gitpitch/services/DiskService.java

@@ -26,11 +26,11 @@ package com.gitpitch.services;
 import com.gitpitch.git.GRS;
 import com.gitpitch.git.GRSService;
 import com.gitpitch.git.GRSManager;
+import com.gitpitch.services.WebService;
 import com.gitpitch.utils.PitchParams;
 import org.apache.commons.io.FileUtils;
 import play.Configuration;
 import play.Logger;
-import play.libs.ws.*;
 
 import javax.inject.*;
 import java.io.File;
@@ -53,12 +53,12 @@ public class DiskService {
     private final String rawAuthToken;
     private final ShellService shellService;
     private final Configuration configuration;
-    private final WSClient ws;
+    private final WebService ws;
 
     @Inject
     public DiskService(ShellService shellService,
                        Configuration configuration,
-                       WSClient ws) {
+                       WebService ws) {
 
         this.shellService = shellService;
         this.configuration = configuration;
@@ -141,44 +141,26 @@ public class DiskService {
 
             final long start = System.currentTimeMillis();
 
-            final WSRequest downloadReq = ws.url(source);
+            byte[] fetched = ws.fetchBytes(pp, source, headers);
 
-            downloadReq.setHeader(API_CACHE_CONTROL, API_NO_CACHE);
+            if (fetched != null) {
 
-            headers.forEach((k,v) -> {
+                Files.write(destPath, fetched);
-                downloadReq.setHeader(k, v);
-            });
-
-            WSResponse downloadResp =
-                    downloadReq.get().toCompletableFuture().get();
-
-            if (downloadResp.getStatus() == HttpURLConnection.HTTP_OK) {
-
-                byte[] body = downloadResp.asByteArray();
-                Files.write(destPath, body);
                 downloaded = STATUS_OK;
                 log.debug("download: pp={}, time-taken={} (ms) to " +
                                 "write {} bytes to {} from source={}", pp,
                         (System.currentTimeMillis() - start),
-                        body.length, destPath, source);
+                        fetched.length, destPath, source);
 
             } else {
-
+                log.debug("download: pp={}, failed to download and write " +
-                downloaded = downloadResp.getStatus();
+                        "from source={}", pp, source);
-                log.debug("download: pp={}, failed status={}, " +
-                        "from source={}", pp, downloaded, source);
             }
-
+        } catch(Exception dex) {
-        } catch (Exception dex) {
             log.warn("download: failed pp={}, from source={}, ex={}",
                     pp, source, dex);
         }
 
-        if (downloaded != STATUS_OK) {
-            log.debug("download: pp={}, from source={}, failed status={}",
-                    pp, source, downloaded);
-        }
-
         return downloaded;
     }
 
@@ -296,8 +278,4 @@ public class DiskService {
     }
 
     private static final Integer STATUS_OK = 0;
-    private static final String API_HEADER_AUTH = "Authorization";
-    private static final String API_HEADER_TOKEN = "token ";
-    private static final String API_CACHE_CONTROL = "Cache-Control";
-    private static final String API_NO_CACHE = "no-cache";
 }

app/com/gitpitch/services/GitService.java

@@ -38,7 +38,6 @@ import play.cache.*;
 import play.libs.ws.*;
 
 import javax.inject.*;
-import java.nio.file.Path;
 import java.util.Optional;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ConcurrentHashMap;
@@ -66,6 +65,7 @@ public class GitService {
     private final CacheTimeout cacheTimeout;
     private final BackEndThreads backEndThreads;
     private final MarkdownModelFactory markdownModelFactory;
+    private final ComposableService composableService;
     private final WSClient wsClient;
     private final CacheApi pitchCache;
     private final Configuration configuration;
@@ -78,6 +78,7 @@ public class GitService {
                       CacheTimeout cacheTimeout,
                       BackEndThreads backEndThreads,
                       MarkdownModelFactory markdownModelFactory,
+                      ComposableService composableService,
                       WSClient wsClient,
                       CacheApi pitchCache,
                       Configuration configuration) {
@@ -89,6 +90,7 @@ public class GitService {
         this.cacheTimeout = cacheTimeout;
         this.backEndThreads = backEndThreads;
         this.markdownModelFactory = markdownModelFactory;
+        this.composableService = composableService;
         this.wsClient = wsClient;
         this.pitchCache = pitchCache;
         this.configuration = configuration;
@@ -323,6 +325,12 @@ public class GitService {
 
                         if (downStatus == STATUS_OK) {
 
+                            /*
+                             * Process Composable Presentation includes
+                             * found within PITCHME.md.
+                             */
+                            composableService.compose(pp, grs, grsService);
+
                             String ssmKey = SlideshowModel.genKey(pp);
                             Optional<SlideshowModel> ssmo =
                                     Optional.ofNullable(pitchCache.get(ssmKey));

app/com/gitpitch/services/WebService.java

@@ -0,0 +1,132 @@
+/*
+ * MIT License
+ *
+ * Copyright (c) 2016 David Russell
+ * 
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ * 
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ * 
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+package com.gitpitch.services;
+
+import com.gitpitch.utils.PitchParams;
+import play.Logger;
+import play.libs.ws.*;
+
+import javax.inject.*;
+import java.net.HttpURLConnection;
+import java.nio.charset.StandardCharsets;
+import java.util.Map;
+
+/*
+ * HTTP(S) web service.
+ */
+@Singleton
+public class WebService {
+
+    private final Logger.ALogger log = Logger.of(this.getClass());
+
+    private final WSClient ws;
+
+    @Inject
+    public WebService(WSClient ws) {
+
+        this.ws = ws;
+    }
+
+    public byte[] fetchBytes(PitchParams pp,
+                             String source,
+                             Map<String,String> headers) {
+
+        byte[] fetched = null;
+
+        try {
+            long start = System.currentTimeMillis();
+
+            WSResponse downloadResp = download(pp, source, headers);
+
+            if (downloadResp.getStatus() == HttpURLConnection.HTTP_OK) {
+
+                fetched = downloadResp.asByteArray();
+                log.debug("fetchBytes: pp={}, time-taken={} (ms) to " +
+                                "read {} bytes from source={}", pp,
+                        (System.currentTimeMillis() - start),
+                        fetched.length, source);
+
+            } else {
+
+                log.debug("fetchBytes: pp={}, failed status={}, " +
+                        "from source={}", pp, downloadResp.getStatus(), source);
+            }
+        } catch(Exception bex) {
+            log.warn("fetchBytes: failed pp={}, from source={}, ex={}",
+                    pp, source, bex);
+        }
+
+        return fetched;
+    }
+
+    public String fetchText(PitchParams pp,
+                            String source,
+                            Map<String,String> headers) {
+
+        byte[] fetched = fetchBytes(pp, source, headers);
+        if(fetched != null)
+            return new String(fetched, StandardCharsets.UTF_8);
+        else
+            return null;
+    }
+
+    /*
+     * Download file from HTTP(s) source url.
+     */
+    private WSResponse download(PitchParams pp,
+                        String source,
+                        Map<String,String> headers) {
+
+        WSResponse downloadResp = null;
+
+        try {
+
+            log.debug("download: pp={}, source={}", pp, source);
+
+            final long start = System.currentTimeMillis();
+
+            final WSRequest downloadReq = ws.url(source);
+
+            downloadReq.setHeader(API_CACHE_CONTROL, API_NO_CACHE);
+
+            headers.forEach((k,v) -> {
+                downloadReq.setHeader(k, v);
+            });
+
+            downloadResp =
+                    downloadReq.get().toCompletableFuture().get();
+
+        } catch (Exception dex) {
+            log.warn("download: failed pp={}, from source={}, ex={}",
+                    pp, source, dex);
+        }
+
+        return downloadResp;
+    }
+
+    private static final String API_HEADER_AUTH = "Authorization";
+    private static final String API_HEADER_TOKEN = "token ";
+    private static final String API_CACHE_CONTROL = "Cache-Control";
+    private static final String API_NO_CACHE = "no-cache";
+}