placeholder expansion in readme and logues; closes #56

also fixes the "scan" volflag which broke in v1.9.14

README.md

@@ -789,6 +789,8 @@ other notes,
 
 * files named `README.md` / `readme.md` will be rendered after directory listings unless `--no-readme` (but `.epilogue.html` takes precedence)
 
+* `README.md` and `*logue.html` can contain placeholder values which are replaced server-side before embedding into directory listings; see `--help-exp`
+
 
 ## searching
 

copyparty/__main__.py

@@ -27,6 +27,7 @@ from .authsrv import expand_config_file, re_vol, split_cfg_ln, upgrade_cfg_fmt
 from .cfg import flagcats, onedash
 from .svchub import SvcHub
 from .util import (
+    DEF_EXP,
     DEF_MTE,
     DEF_MTH,
     IMPLICATIONS,
@@ -646,6 +647,47 @@ def get_sects():
             """
             ),
         ],
+        [
+            "exp",
+            "text expansion",
+            dedent(
+                """
+            specify --exp or the "exp" volflag to enable placeholder expansions
+            in README.md / .prologue.html / .epilogue.html
+
+            --exp-md (volflag exp_md) holds the list of placeholders which can be
+            expanded in READMEs, and --exp-lg (volflag exp_lg) likewise for logues;
+            any placeholder not given in those lists will be ignored and shown as-is
+
+            the default list will expand the following placeholders:
+            \033[36m{{self.ip}}     \033[35mclient ip
+            \033[36m{{self.ua}}     \033[35mclient user-agent
+            \033[36m{{self.uname}}  \033[35mclient username
+            \033[36m{{self.host}}   \033[35mthe "Host" header, or the server's external IP otherwise
+            \033[36m{{cfg.name}}    \033[35mthe --name global-config
+            \033[36m{{cfg.logout}}  \033[35mthe --logout global-config
+            \033[36m{{vf.scan}}     \033[35mthe "scan" volflag
+            \033[36m{{vf.thsize}}   \033[35mthumbnail size
+            \033[36m{{srv.itime}}   \033[35mserver time in seconds
+            \033[36m{{srv.htime}}   \033[35mserver time as YY-mm-dd, HH:MM:SS (UTC)
+            \033[36m{{hdr.cf_ipcountry}} \033[35mthe "CF-IPCountry" client header (probably blank)
+            \033[0m
+            so the following types of placeholders can be added to the lists:
+            * any client header can be accessed through {{hdr.*}}
+            * any variable in httpcli.py can be accessed through {{self.*}}
+            * any global server setting can be accessed through {{cfg.*}}
+            * any volflag can be accessed through {{vf.*}}
+
+            remove vf.scan from default list using --exp-md /vf.scan
+            add "accept" header to def. list using --exp-md +hdr.accept
+
+            for performance reasons, expansion only happens while embedding
+            documents into directory listings, and when accessing a ?doc=...
+            link, but never otherwise, so if you click a -txt- link you'll
+            have to refresh the page to apply expansion
+            """
+            ),
+        ],
         [
             "ls",
             "volume inspection",
@@ -776,8 +818,6 @@ def add_general(ap, nc, srvname):
     ap2.add_argument("-a", metavar="ACCT", type=u, action="append", help="add account, \033[33mUSER\033[0m:\033[33mPASS\033[0m; example [\033[32med:wark\033[0m]")
     ap2.add_argument("-v", metavar="VOL", type=u, action="append", help="add volume, \033[33mSRC\033[0m:\033[33mDST\033[0m:\033[33mFLAG\033[0m; examples [\033[32m.::r\033[0m], [\033[32m/mnt/nas/music:/music:r:aed\033[0m]")
     ap2.add_argument("-ed", action="store_true", help="enable the ?dots url parameter / client option which allows clients to see dotfiles / hidden files")
-    ap2.add_argument("-emp", action="store_true", help="enable markdown plugins -- neat but dangerous, big XSS risk")
-    ap2.add_argument("-mcr", metavar="SEC", type=int, default=60, help="md-editor mod-chk rate")
     ap2.add_argument("--urlform", metavar="MODE", type=u, default="print,get", help="how to handle url-form POSTs; see --help-urlform")
     ap2.add_argument("--wintitle", metavar="TXT", type=u, default="cpp @ $pub", help="window title, for example [\033[32m$ip-10.1.2.\033[0m] or [\033[32m$ip-]")
     ap2.add_argument("--name", metavar="TXT", type=u, default=srvname, help="server name (displayed topleft in browser and in mDNS)")
@@ -1144,6 +1184,15 @@ def add_db_metadata(ap):
     ap2.add_argument("-mtp", metavar="M=[f,]BIN", type=u, action="append", help="read tag M using program BIN to parse the file")
 
 
+def add_txt(ap):
+    ap2 = ap.add_argument_group('textfile options')
+    ap2.add_argument("-mcr", metavar="SEC", type=int, default=60, help="textfile editor checks for serverside changes every SEC seconds")
+    ap2.add_argument("-emp", action="store_true", help="enable markdown plugins -- neat but dangerous, big XSS risk")
+    ap2.add_argument("--exp", action="store_true", help="enable textfile expansion -- replace {{self.ip}} and such; see --help-exp (volflag=exp)")
+    ap2.add_argument("--exp-md", metavar="V,V,V", type=u, default=DEF_EXP, help="comma/space-separated list of placeholders to expand in markdown files; add/remove stuff on the default list with +hdr_foo or /vf.scan (volflag=exp_md)")
+    ap2.add_argument("--exp-lg", metavar="V,V,V", type=u, default=DEF_EXP, help="comma/space-separated list of placeholders to expand in prologue/epilogue files (volflag=exp_lg)")
+
+
 def add_ui(ap, retry):
     ap2 = ap.add_argument_group('ui options')
     ap2.add_argument("--grid", action="store_true", help="show grid/thumbnails by default (volflag=grid)")
@@ -1237,6 +1286,7 @@ def run_argparse(
     add_handlers(ap)
     add_hooks(ap)
     add_stats(ap)
+    add_txt(ap)
     add_ui(ap, retry)
     add_admin(ap)
     add_logging(ap)

copyparty/authsrv.py

@@ -1479,15 +1479,11 @@ class AuthSrv(object):
                 raise Exception(t.format(dbd, dbds))
 
             # default tag cfgs if unset
-            if "mte" not in vol.flags:
-                vol.flags["mte"] = self.args.mte.copy()
-            else:
-                vol.flags["mte"] = odfusion(self.args.mte, vol.flags["mte"])
-
-            if "mth" not in vol.flags:
-                vol.flags["mth"] = self.args.mth.copy()
-            else:
-                vol.flags["mth"] = odfusion(self.args.mth, vol.flags["mth"])
+            for k in ("mte", "mth", "exp_md", "exp_lg"):
+                if k not in vol.flags:
+                    vol.flags[k] = getattr(self.args, k).copy()
+                else:
+                    vol.flags[k] = odfusion(getattr(self.args, k), vol.flags[k])
 
             # append additive args from argv to volflags
             hooks = "xbu xau xiu xbr xar xbd xad xm xban".split()

copyparty/cfg.py

@@ -17,7 +17,6 @@ def vf_bmap() -> dict[str, str]:
         "no_thumb": "dthumb",
         "no_vthumb": "dvthumb",
         "no_athumb": "dathumb",
-        "re_maxage": "scan",
         "th_no_crop": "nocrop",
         "dav_auth": "davauth",
         "dav_rt": "davrt",
@@ -33,6 +32,7 @@ def vf_bmap() -> dict[str, str]:
         "e2v",
         "e2vu",
         "e2vp",
+        "exp",
         "grid",
         "hardlink",
         "magic",
@@ -52,10 +52,19 @@ def vf_vmap() -> dict[str, str]:
     ret = {
         "no_hash": "nohash",
         "no_idx": "noidx",
+        "re_maxage": "scan",
         "th_convt": "convt",
         "th_size": "thsize",
     }
-    for k in ("dbd", "lg_sbf", "md_sbf", "nrand", "sort", "unlist", "u2ts"):
+    for k in (
+        "dbd",
+        "lg_sbf",
+        "md_sbf",
+        "nrand",
+        "sort",
+        "unlist",
+        "u2ts",
+    ):
         ret[k] = k
     return ret
 
@@ -64,6 +73,8 @@ def vf_cmap() -> dict[str, str]:
     """argv-to-volflag: complex/lists"""
     ret = {}
     for k in (
+        "exp_lg",
+        "exp_md",
         "html_head",
         "mte",
         "mth",

copyparty/httpcli.py

@@ -2726,6 +2726,29 @@ class HttpCli(object):
 
         return file_lastmod, True
 
+    def _expand(self, txt: str, phs: list[str]) -> str:
+        for ph in phs:
+            if ph.startswith("hdr."):
+                sv = str(self.headers.get(ph[4:], ""))
+            elif ph.startswith("self."):
+                sv = str(getattr(self, ph[5:], ""))
+            elif ph.startswith("cfg."):
+                sv = str(getattr(self.args, ph[4:], ""))
+            elif ph.startswith("vf."):
+                sv = str(self.vn.flags.get(ph[3:]) or "")
+            elif ph == "srv.itime":
+                sv = str(int(time.time()))
+            elif ph == "srv.htime":
+                sv = datetime.now(UTC).strftime("%Y-%m-%d, %H:%M:%S")
+            else:
+                self.log("unknown placeholder in server config: [%s]" % (ph), 3)
+                continue
+
+            sv = self.conn.hsrv.ptn_hsafe.sub("_", sv)
+            txt = txt.replace("{{%s}}" % (ph,), sv)
+
+        return txt
+
     def tx_file(self, req_path: str) -> bool:
         status = 200
         logmsg = "{:4} {} ".format("", self.req)
@@ -3052,7 +3075,7 @@ class HttpCli(object):
         self.reply(ico, mime=mime, headers={"Last-Modified": lm})
         return True
 
-    def tx_md(self, fs_path: str) -> bool:
+    def tx_md(self, vn: VFS, fs_path: str) -> bool:
         logmsg = "     %s @%s " % (self.req, self.uname)
 
         if not self.can_write:
@@ -3069,9 +3092,16 @@ class HttpCli(object):
         st = bos.stat(html_path)
         ts_html = st.st_mtime
 
+        max_sz = 1024 * self.args.txt_max
         sz_md = 0
         lead = b""
+        fullfile = b""
         for buf in yieldfile(fs_path):
+            if sz_md < max_sz:
+                fullfile += buf
+            else:
+                fullfile = b""
+
             if not sz_md and b"\n" in buf[:2]:
                 lead = buf[: buf.find(b"\n") + 1]
                 sz_md += len(lead)
@@ -3080,6 +3110,21 @@ class HttpCli(object):
             for c, v in [(b"&", 4), (b"<", 3), (b">", 3)]:
                 sz_md += (len(buf) - len(buf.replace(c, b""))) * v
 
+        if (
+            fullfile
+            and "exp" in vn.flags
+            and "edit" not in self.uparam
+            and "edit2" not in self.uparam
+            and vn.flags.get("exp_md")
+        ):
+            fulltxt = fullfile.decode("utf-8", "replace")
+            fulltxt = self._expand(fulltxt, vn.flags.get("exp_md") or [])
+            fullfile = fulltxt.encode("utf-8", "replace")
+
+        if fullfile:
+            fullfile = html_bescape(fullfile)
+            sz_md = len(lead) + len(fullfile)
+
         file_ts = int(max(ts_md, ts_html, self.E.t0))
         file_lastmod, do_send = self._chk_lastmod(file_ts)
         self.out_headers["Last-Modified"] = file_lastmod
@@ -3121,8 +3166,11 @@ class HttpCli(object):
 
         try:
             self.s.sendall(html[0] + lead)
-            for buf in yieldfile(fs_path):
-                self.s.sendall(html_bescape(buf))
+            if fullfile:
+                self.s.sendall(fullfile)
+            else:
+                for buf in yieldfile(fs_path):
+                    self.s.sendall(html_bescape(buf))
 
             self.s.sendall(html[1])
 
@@ -3753,7 +3801,7 @@ class HttpCli(object):
                     or "edit2" in self.uparam
                 )
             ):
-                return self.tx_md(abspath)
+                return self.tx_md(vn, abspath)
 
             return self.tx_file(abspath)
 
@@ -3815,6 +3863,10 @@ class HttpCli(object):
                 if bos.path.exists(fn):
                     with open(fsenc(fn), "rb") as f:
                         logues[n] = f.read().decode("utf-8")
+                    if "exp" in vn.flags:
+                        logues[n] = self._expand(
+                            logues[n], vn.flags.get("exp_lg") or []
+                        )
 
         readme = ""
         if not self.args.no_readme and not logues[1]:
@@ -3824,6 +3876,8 @@ class HttpCli(object):
                     with open(fsenc(fn), "rb") as f:
                         readme = f.read().decode("utf-8")
                         break
+            if readme and "exp" in vn.flags:
+                readme = self._expand(readme, vn.flags.get("exp_md") or [])
 
         vf = vn.flags
         unlist = vf.get("unlist", "")
@@ -4134,6 +4188,12 @@ class HttpCli(object):
                 if sz < 1024 * self.args.txt_max:
                     with open(fsenc(docpath), "rb") as f:
                         doctxt = f.read().decode("utf-8", "replace")
+
+                    if doc.lower().endswith(".md") and "exp" in vn.flags:
+                        doctxt = self._expand(doctxt, vn.flags.get("exp_md") or [])
+                else:
+                    self.log("doc 2big: [{}]".format(doc), c=6)
+                    doctxt = "( size of textfile exceeds serverside limit )"
             else:
                 self.log("doc 404: [{}]".format(doc), c=6)
                 doctxt = "( textfile not found )"

copyparty/httpsrv.py

@@ -149,6 +149,7 @@ class HttpSrv(object):
         self._build_statics()
 
         self.ptn_cc = re.compile(r"[\x00-\x1f]")
+        self.ptn_hsafe = re.compile(r"[\x00-\x1f<>\"'&]")
 
         self.mallow = "GET HEAD POST PUT DELETE OPTIONS".split()
         if not self.args.no_dav:

copyparty/svchub.py

@@ -39,6 +39,7 @@ from .util import (
     FFMPEG_URL,
     VERSIONS,
     Daemon,
+    DEF_EXP,
     DEF_MTE,
     DEF_MTH,
     Garda,
@@ -442,6 +443,10 @@ class SvcHub(object):
         mth = ODict.fromkeys(DEF_MTH.split(","), True)
         al.mth = odfusion(mth, al.mth)
 
+        exp = ODict.fromkeys(DEF_EXP.split(" "), True)
+        al.exp_md = odfusion(exp, al.exp_md.replace(" ", ","))
+        al.exp_lg = odfusion(exp, al.exp_lg.replace(" ", ","))
+
         for k in ["no_hash", "no_idx"]:
             ptn = getattr(self.args, k)
             if ptn:

copyparty/up2k.py

@@ -814,7 +814,7 @@ class Up2k(object):
             if str(fl[k1]) == str(getattr(self.args, k2)):
                 del fl[k1]
             else:
-                fl[k1] = ",".join(x for x in fl)
+                fl[k1] = ",".join(x for x in fl[k1])
         a = [
             (ft if v is True else ff if v is False else fv).format(k, str(v))
             for k, v in fl.items()

copyparty/util.py

@@ -289,6 +289,8 @@ EXTS["vnd.mozilla.apng"] = "png"
 MAGIC_MAP = {"jpeg": "jpg"}
 
 
+DEF_EXP = "self.ip self.ua self.uname self.host cfg.name cfg.logout vf.scan vf.thsize hdr.cf_ipcountry srv.itime srv.htime"
+
 DEF_MTE = "circle,album,.tn,artist,title,.bpm,key,.dur,.q,.vq,.aq,vc,ac,fmt,res,.fps,ahash,vhash"
 
 DEF_MTH = ".vq,.aq,vc,ac,fmt,res,.fps"
@@ -1809,15 +1811,18 @@ def exclude_dotfiles(filepaths: list[str]) -> list[str]:
 
 def odfusion(base: ODict[str, bool], oth: str) -> ODict[str, bool]:
     # merge an "ordered set" (just a dict really) with another list of keys
+    words0 = [x for x in oth.split(",") if x]
+    words1 = [x for x in oth[1:].split(",") if x]
+
     ret = base.copy()
     if oth.startswith("+"):
-        for k in oth[1:].split(","):
+        for k in words1:
             ret[k] = True
     elif oth[:1] in ("-", "/"):
-        for k in oth[1:].split(","):
+        for k in words1:
             ret.pop(k, None)
     else:
-        ret = ODict.fromkeys(oth.split(","), True)
+        ret = ODict.fromkeys(words0, True)
 
     return ret
 

srv/expand/README.md

@@ -0,0 +1,26 @@
+## text expansion
+
+enable expansion of placeholder variables in `README.md` and prologue/epilogue files with `--exp` and customize the list of allowed placeholders to expand using `--exp-md` and `--exp-lg`
+
+| explanation          | placeholder          |
+| -------------------- | -------------------- |
+| your ip address      | {{self.ip}}          |
+| your user-agent      | {{self.ua}}          |
+| your username        | {{self.uname}}       |
+| the `Host` you see   | {{self.host}}        |
+| server unix time     | {{srv.itime}}        |
+| server datetime      | {{srv.htime}}        |
+| server name          | {{cfg.name}}         |
+| logout after         | {{cfg.logout}} hours |
+| vol reindex interval | {{vf.scan}}          |
+| thumbnail size       | {{vf.thsize}}        |
+| your country         | {{hdr.cf_ipcountry}} |
+
+placeholders starting with...
+* `self.` are grabbed from copyparty's internal state; anything in `httpcli.py` is fair game
+* `cfg.` are the global server settings
+* `vf.` are the volflags of the current volume
+* `hdr.` are grabbed from the client headers; any header is supported, just add it (in lowercase) to the allowlist
+* `srv.` are processed inside the `_expand` function in httpcli
+
+for example (bad example), `hdr_cf_ipcountry` maps to the header `CF-IPCountry` (which is generated by cloudflare before the request is passed on to your server / copyparty)

tests/util.py

@@ -109,7 +109,7 @@ class Cfg(Namespace):
     def __init__(self, a=None, v=None, c=None):
         ka = {}
 
-        ex = "daw dav_auth dav_inf dav_mac dav_rt dotsrch e2d e2ds e2dsa e2t e2ts e2tsr e2v e2vu e2vp ed emp force_js getmod grid hardlink ih ihead magic never_symlink nid nih no_acode no_athumb no_dav no_dedup no_del no_dupe no_logues no_mv no_readme no_robots no_sb_md no_sb_lg no_scandir no_tarcmp no_thumb no_vthumb no_zip nrand nw rand smb th_no_crop vague_403 vc ver xdev xlink xvol"
+        ex = "daw dav_auth dav_inf dav_mac dav_rt dotsrch e2d e2ds e2dsa e2t e2ts e2tsr e2v e2vu e2vp ed emp exp force_js getmod grid hardlink ih ihead magic never_symlink nid nih no_acode no_athumb no_dav no_dedup no_del no_dupe no_logues no_mv no_readme no_robots no_sb_md no_sb_lg no_scandir no_tarcmp no_thumb no_vthumb no_zip nrand nw rand smb th_no_crop vague_403 vc ver xdev xlink xvol"
         ka.update(**{k: False for k in ex.split()})
 
         ex = "dotpart no_rescan no_sendfile no_voldump plain_ip"
@@ -130,6 +130,9 @@ class Cfg(Namespace):
         ex = "on403 on404 xad xar xau xban xbd xbr xbu xiu xm"
         ka.update(**{k: [] for k in ex.split()})
 
+        ex = "exp_lg exp_md"
+        ka.update(**{k: {} for k in ex.split()})
+
         super(Cfg, self).__init__(
             a=a or [],
             v=v or [],