From 2b11bc310f255cee4ca2e4997b6060f7809fd63c Mon Sep 17 00:00:00 2001 From: David Russell Date: Sat, 18 Mar 2017 17:19:40 +0700 Subject: [PATCH] Added support for modular, composable presentation markdown. --- app/Module.java | 2 + .../gitpitch/services/ComposableService.java | 227 ++++++++++++++++++ app/com/gitpitch/services/DiskService.java | 42 +--- app/com/gitpitch/services/GitService.java | 10 +- app/com/gitpitch/services/WebService.java | 132 ++++++++++ 5 files changed, 380 insertions(+), 33 deletions(-) create mode 100644 app/com/gitpitch/services/ComposableService.java create mode 100644 app/com/gitpitch/services/WebService.java diff --git a/app/Module.java b/app/Module.java index 1e53609..2948ac8 100644 --- a/app/Module.java +++ b/app/Module.java @@ -51,6 +51,8 @@ public class Module extends AbstractModule { bind(ImageService.class).asEagerSingleton(); bind(VideoService.class).asEagerSingleton(); bind(GISTService.class).asEagerSingleton(); + bind(WebService.class).asEagerSingleton(); + bind(ComposableService.class).asEagerSingleton(); bind(GRSManager.class).asEagerSingleton(); bind(GitHub.class).asEagerSingleton(); bind(GitLab.class).asEagerSingleton(); diff --git a/app/com/gitpitch/services/ComposableService.java b/app/com/gitpitch/services/ComposableService.java new file mode 100644 index 0000000..594ad02 --- /dev/null +++ b/app/com/gitpitch/services/ComposableService.java @@ -0,0 +1,227 @@ +/* + * MIT License + * + * Copyright (c) 2016 David Russell + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package com.gitpitch.services; + +import com.gitpitch.utils.PitchParams; +import com.gitpitch.git.GRS; +import com.gitpitch.git.GRSService; +import com.gitpitch.services.WebService; +import com.gitpitch.models.MarkdownModel; +import play.Logger; + +import javax.inject.*; +import java.net.HttpURLConnection; +import java.nio.charset.StandardCharsets; +import java.util.Map; +import java.util.Arrays; +import java.util.List; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +/* + * Composable service supporting Markdown includes in + * the form of: + * + * ---?include=/path/to/some.md + * +++?include=/path/to/other.md + */ +@Singleton +public class ComposableService { + + private final Logger.ALogger log = Logger.of(this.getClass()); + + private final DiskService ds; + private final WebService ws; + + @Inject + public ComposableService(DiskService ds, + WebService ws) { + + this.ds = ds; + this.ws = ws; + } + + /* + * Make best effort to process composable presentation + * includes within PITCHME.md. + */ + public void compose(PitchParams pp, + GRS grs, + GRSService grsService) { + + try { + + Path mdPath = ds.asPath(pp, PitchParams.PITCHME_MD); + + String composed = null; + try (Stream stream = Files.lines(mdPath)) { + + composed = stream.map(md -> { + + if(md.startsWith(HSLIDE_INCLUDE)) { + + String included = handleInclude(pp, + md, + HSLIDE_INCLUDE, + grsService, + grs.getHeaders()); + if(included != null) { + return MarkdownModel.HSLIDE_DELIM_DEFAULT + + NEWLINES + included; + } else { + return notFound(MarkdownModel.HSLIDE_DELIM_DEFAULT, + includePath(md, HSLIDE_INCLUDE)); + } + } else + if(md.startsWith(VSLIDE_INCLUDE)) { + + String included = handleInclude(pp, + md, + VSLIDE_INCLUDE, + grsService, + grs.getHeaders()); + if(included != null) { + return MarkdownModel.VSLIDE_DELIM_DEFAULT + + NEWLINES + included; + } else { + return notFound(MarkdownModel.VSLIDE_DELIM_DEFAULT, + includePath(md, VSLIDE_INCLUDE)); + } + } else { + return md; + } + }).collect(Collectors.joining("\n")); + + boolean overwritten = overwrite(pp, mdPath, composed); + log.debug("compose: overwritten md={}", overwritten); + + } catch (Exception mex) { + log.warn("Markdown processing ex={}", mex); + composed = "PITCHME.md could not be parsed."; + } + + } catch(Exception stex) { + + } + } + + private String handleInclude(PitchParams pp, + String md, + String includeDelim, + GRSService grsService, + Map headers) { + + String included = null; + String includePath = includePath(md, includeDelim); + + if(FORBIDDEN.contains(includePath)) { + included = notPermitted(includeDelim, includePath); + } else { + included = fetchInclude(pp, includePath, + includeDelim, grsService, headers); + } + return included; + } + + private String fetchInclude(PitchParams pp, + String includePath, + String includeDelim, + GRSService grsService, + Map headers) { + + String fetched = null; + + try { + + if(!isAbsolute(includePath)) + includePath = grsService.raw(pp, includePath, true); + + fetched = ws.fetchText(pp, includePath, headers); + + } catch(Exception fiex) { + log.warn("fetchInclude: pp={}, error={}", pp, fiex); + } + + return fetched; + } + + private String includePath(String md, String includeDelim) { + try { + return md.substring(includeDelim.length()); + } catch(Exception ipex) { + return md; + } + } + + private boolean overwrite(PitchParams pp, + Path mdPath, + String stitched) { + try { + Files.write(mdPath, stitched.getBytes(), + StandardOpenOption.CREATE, + StandardOpenOption.TRUNCATE_EXISTING ); + return true; + } catch(Exception fex) { + log.warn("overwrite: pp={}, write error={}", pp, fex); + return false; + } + } + + private String notFound(String delim, String path) { + StringBuffer buf = new StringBuffer(delim); + buf.append(NEWLINES) + .append("### GitPitch ?Include") + .append(NEWLINES).append(".").append(NEWLINES) + .append(path) + .append(NEWLINES).append(".").append(NEWLINES) + .append("Markdown File Not Found"); + return buf.toString(); + } + + private String notPermitted(String delim, String path) { + StringBuffer buf = new StringBuffer(NEWLINES); + buf.append("### GitPitch ?Include") + .append(NEWLINES).append(".").append(NEWLINES) + .append(path) + .append(NEWLINES).append(".").append(NEWLINES) + .append("Path Not Permitted"); + return buf.toString(); + } + + private boolean isAbsolute(String path) { + return path.startsWith(HTTP); + } + + public final String HSLIDE_INCLUDE = + MarkdownModel.HSLIDE_DELIM_DEFAULT + "?include="; + public final String VSLIDE_INCLUDE = + MarkdownModel.VSLIDE_DELIM_DEFAULT + "?include="; + private final String HTTP = "http"; + private final String NEWLINES = "\n\n"; + private final List FORBIDDEN = + Arrays.asList(PitchParams.PITCHME_MD, "./PITCHME.md"); +} diff --git a/app/com/gitpitch/services/DiskService.java b/app/com/gitpitch/services/DiskService.java index 8baafb8..7b16ba3 100644 --- a/app/com/gitpitch/services/DiskService.java +++ b/app/com/gitpitch/services/DiskService.java @@ -26,11 +26,11 @@ package com.gitpitch.services; import com.gitpitch.git.GRS; import com.gitpitch.git.GRSService; import com.gitpitch.git.GRSManager; +import com.gitpitch.services.WebService; import com.gitpitch.utils.PitchParams; import org.apache.commons.io.FileUtils; import play.Configuration; import play.Logger; -import play.libs.ws.*; import javax.inject.*; import java.io.File; @@ -53,12 +53,12 @@ public class DiskService { private final String rawAuthToken; private final ShellService shellService; private final Configuration configuration; - private final WSClient ws; + private final WebService ws; @Inject public DiskService(ShellService shellService, Configuration configuration, - WSClient ws) { + WebService ws) { this.shellService = shellService; this.configuration = configuration; @@ -141,44 +141,26 @@ public class DiskService { final long start = System.currentTimeMillis(); - final WSRequest downloadReq = ws.url(source); + byte[] fetched = ws.fetchBytes(pp, source, headers); - downloadReq.setHeader(API_CACHE_CONTROL, API_NO_CACHE); + if (fetched != null) { - headers.forEach((k,v) -> { - downloadReq.setHeader(k, v); - }); - - WSResponse downloadResp = - downloadReq.get().toCompletableFuture().get(); - - if (downloadResp.getStatus() == HttpURLConnection.HTTP_OK) { - - byte[] body = downloadResp.asByteArray(); - Files.write(destPath, body); + Files.write(destPath, fetched); downloaded = STATUS_OK; log.debug("download: pp={}, time-taken={} (ms) to " + "write {} bytes to {} from source={}", pp, (System.currentTimeMillis() - start), - body.length, destPath, source); + fetched.length, destPath, source); } else { - - downloaded = downloadResp.getStatus(); - log.debug("download: pp={}, failed status={}, " + - "from source={}", pp, downloaded, source); + log.debug("download: pp={}, failed to download and write " + + "from source={}", pp, source); } - - } catch (Exception dex) { + } catch(Exception dex) { log.warn("download: failed pp={}, from source={}, ex={}", pp, source, dex); } - if (downloaded != STATUS_OK) { - log.debug("download: pp={}, from source={}, failed status={}", - pp, source, downloaded); - } - return downloaded; } @@ -296,8 +278,4 @@ public class DiskService { } private static final Integer STATUS_OK = 0; - private static final String API_HEADER_AUTH = "Authorization"; - private static final String API_HEADER_TOKEN = "token "; - private static final String API_CACHE_CONTROL = "Cache-Control"; - private static final String API_NO_CACHE = "no-cache"; } diff --git a/app/com/gitpitch/services/GitService.java b/app/com/gitpitch/services/GitService.java index a88d9f0..6efc31c 100644 --- a/app/com/gitpitch/services/GitService.java +++ b/app/com/gitpitch/services/GitService.java @@ -38,7 +38,6 @@ import play.cache.*; import play.libs.ws.*; import javax.inject.*; -import java.nio.file.Path; import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; @@ -66,6 +65,7 @@ public class GitService { private final CacheTimeout cacheTimeout; private final BackEndThreads backEndThreads; private final MarkdownModelFactory markdownModelFactory; + private final ComposableService composableService; private final WSClient wsClient; private final CacheApi pitchCache; private final Configuration configuration; @@ -78,6 +78,7 @@ public class GitService { CacheTimeout cacheTimeout, BackEndThreads backEndThreads, MarkdownModelFactory markdownModelFactory, + ComposableService composableService, WSClient wsClient, CacheApi pitchCache, Configuration configuration) { @@ -89,6 +90,7 @@ public class GitService { this.cacheTimeout = cacheTimeout; this.backEndThreads = backEndThreads; this.markdownModelFactory = markdownModelFactory; + this.composableService = composableService; this.wsClient = wsClient; this.pitchCache = pitchCache; this.configuration = configuration; @@ -323,6 +325,12 @@ public class GitService { if (downStatus == STATUS_OK) { + /* + * Process Composable Presentation includes + * found within PITCHME.md. + */ + composableService.compose(pp, grs, grsService); + String ssmKey = SlideshowModel.genKey(pp); Optional ssmo = Optional.ofNullable(pitchCache.get(ssmKey)); diff --git a/app/com/gitpitch/services/WebService.java b/app/com/gitpitch/services/WebService.java new file mode 100644 index 0000000..859ff4f --- /dev/null +++ b/app/com/gitpitch/services/WebService.java @@ -0,0 +1,132 @@ +/* + * MIT License + * + * Copyright (c) 2016 David Russell + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package com.gitpitch.services; + +import com.gitpitch.utils.PitchParams; +import play.Logger; +import play.libs.ws.*; + +import javax.inject.*; +import java.net.HttpURLConnection; +import java.nio.charset.StandardCharsets; +import java.util.Map; + +/* + * HTTP(S) web service. + */ +@Singleton +public class WebService { + + private final Logger.ALogger log = Logger.of(this.getClass()); + + private final WSClient ws; + + @Inject + public WebService(WSClient ws) { + + this.ws = ws; + } + + public byte[] fetchBytes(PitchParams pp, + String source, + Map headers) { + + byte[] fetched = null; + + try { + long start = System.currentTimeMillis(); + + WSResponse downloadResp = download(pp, source, headers); + + if (downloadResp.getStatus() == HttpURLConnection.HTTP_OK) { + + fetched = downloadResp.asByteArray(); + log.debug("fetchBytes: pp={}, time-taken={} (ms) to " + + "read {} bytes from source={}", pp, + (System.currentTimeMillis() - start), + fetched.length, source); + + } else { + + log.debug("fetchBytes: pp={}, failed status={}, " + + "from source={}", pp, downloadResp.getStatus(), source); + } + } catch(Exception bex) { + log.warn("fetchBytes: failed pp={}, from source={}, ex={}", + pp, source, bex); + } + + return fetched; + } + + public String fetchText(PitchParams pp, + String source, + Map headers) { + + byte[] fetched = fetchBytes(pp, source, headers); + if(fetched != null) + return new String(fetched, StandardCharsets.UTF_8); + else + return null; + } + + /* + * Download file from HTTP(s) source url. + */ + private WSResponse download(PitchParams pp, + String source, + Map headers) { + + WSResponse downloadResp = null; + + try { + + log.debug("download: pp={}, source={}", pp, source); + + final long start = System.currentTimeMillis(); + + final WSRequest downloadReq = ws.url(source); + + downloadReq.setHeader(API_CACHE_CONTROL, API_NO_CACHE); + + headers.forEach((k,v) -> { + downloadReq.setHeader(k, v); + }); + + downloadResp = + downloadReq.get().toCompletableFuture().get(); + + } catch (Exception dex) { + log.warn("download: failed pp={}, from source={}, ex={}", + pp, source, dex); + } + + return downloadResp; + } + + private static final String API_HEADER_AUTH = "Authorization"; + private static final String API_HEADER_TOKEN = "token "; + private static final String API_CACHE_CONTROL = "Cache-Control"; + private static final String API_NO_CACHE = "no-cache"; +}