aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorkj_sh6042026-04-18 14:25:31 -0400
committerkj_sh6042026-04-18 14:25:31 -0400
commitf4b6ffce91bcf9cd3c4ef85dd1a2b30b043c2d4d (patch)
tree3630313212ce37e90311e5f1fd3bfac5c0d5a566
parentabf56914d2e9b2658b29be96811930ceedac9aa7 (diff)
refactor: better concurrency and use base highlight options
-rw-r--r--src/index.html324
-rw-r--r--src/main.js219
-rw-r--r--src/server.py430
3 files changed, 437 insertions, 536 deletions
diff --git a/src/index.html b/src/index.html
index 8d89859..f5510a2 100644
--- a/src/index.html
+++ b/src/index.html
@@ -96,329 +96,7 @@
<div class="controls"><button id="get-link-btn" type="button" title="generate snippet link">generate
link</button><span class="spacer"></span><label><input type="checkbox" id="is-code">is this code?
</label><select id="lang-select">
- <option value="">auto-detect</option>
- <option value="1c">1c</option>
- <option value="abnf">abnf</option>
- <option value="accesslog">accesslog</option>
- <option value="actionscript">actionscript</option>
- <option value="as">as</option>
- <option value="ada">ada</option>
- <option value="angelscript">angelscript</option>
- <option value="asc">asc</option>
- <option value="apache">apache</option>
- <option value="apacheconf">apacheconf</option>
- <option value="applescript">applescript</option>
- <option value="osascript">osascript</option>
- <option value="arcade">arcade</option>
- <option value="arduino">arduino</option>
- <option value="ino">ino</option>
- <option value="armasm">armasm</option>
- <option value="arm">arm</option>
- <option value="asciidoc">asciidoc</option>
- <option value="adoc">adoc</option>
- <option value="aspectj">aspectj</option>
- <option value="autohotkey">autohotkey</option>
- <option value="autoit">autoit</option>
- <option value="avrasm">avrasm</option>
- <option value="awk">awk</option>
- <option value="mawk">mawk</option>
- <option value="nawk">nawk</option>
- <option value="gawk">gawk</option>
- <option value="ballerina">ballerina</option>
- <option value="bal">bal</option>
- <option value="bash">bash</option>
- <option value="sh">sh</option>
- <option value="zsh">zsh</option>
- <option value="basic">basic</option>
- <option value="bnf">bnf</option>
- <option value="brainfuck">brainfuck</option>
- <option value="bf">bf</option>
- <option value="c">c</option>
- <option value="h">h</option>
- <option value="csharp">csharp</option>
- <option value="cpp">cpp</option>
- <option value="hpp">hpp</option>
- <option value="cc">cc</option>
- <option value="hh">hh</option>
- <option value="c++">c++</option>
- <option value="h++">h++</option>
- <option value="cxx">cxx</option>
- <option value="hxx">hxx</option>
- <option value="cal">cal</option>
- <option value="cos">cos</option>
- <option value="cls">cls</option>
- <option value="capnproto">capnproto</option>
- <option value="capnp">capnp</option>
- <option value="clojure">clojure</option>
- <option value="clj">clj</option>
- <option value="cmake">cmake</option>
- <option value="cmake.in">cmake.in</option>
- <option value="coffeescript">coffeescript</option>
- <option value="coffee">coffee</option>
- <option value="cson">cson</option>
- <option value="iced">iced</option>
- <option value="coq">coq</option>
- <option value="crmsh">crmsh</option>
- <option value="crm">crm</option>
- <option value="pcmk">pcmk</option>
- <option value="crystal">crystal</option>
- <option value="cr">cr</option>
- <option value="csp">csp</option>
- <option value="css">css</option>
- <option value="d">d</option>
- <option value="dart">dart</option>
- <option value="dpr">dpr</option>
- <option value="dfm">dfm</option>
- <option value="pas">pas</option>
- <option value="pascal">pascal</option>
- <option value="diff">diff</option>
- <option value="patch">patch</option>
- <option value="django">django</option>
- <option value="jinja">jinja</option>
- <option value="dns">dns</option>
- <option value="zone">zone</option>
- <option value="bind">bind</option>
- <option value="dockerfile">dockerfile</option>
- <option value="docker">docker</option>
- <option value="dos">dos</option>
- <option value="bat">bat</option>
- <option value="cmd">cmd</option>
- <option value="dsconfig">dsconfig</option>
- <option value="dts">dts</option>
- <option value="dust">dust</option>
- <option value="dst">dst</option>
- <option value="ebnf">ebnf</option>
- <option value="elixir">elixir</option>
- <option value="elm">elm</option>
- <option value="erlang">erlang</option>
- <option value="erl">erl</option>
- <option value="excel">excel</option>
- <option value="xls">xls</option>
- <option value="xlsx">xlsx</option>
- <option value="extempore">extempore</option>
- <option value="xtlang">xtlang</option>
- <option value="xtm">xtm</option>
- <option value="fsharp">fsharp</option>
- <option value="fs">fs</option>
- <option value="fsx">fsx</option>
- <option value="fsi">fsi</option>
- <option value="fsscript">fsscript</option>
- <option value="fix">fix</option>
- <option value="fortran">fortran</option>
- <option value="f90">f90</option>
- <option value="f95">f95</option>
- <option value="gcode">gcode</option>
- <option value="nc">nc</option>
- <option value="gams">gams</option>
- <option value="gms">gms</option>
- <option value="gauss">gauss</option>
- <option value="gss">gss</option>
- <option value="gherkin">gherkin</option>
- <option value="go">go</option>
- <option value="golang">golang</option>
- <option value="golo">golo</option>
- <option value="gololang">gololang</option>
- <option value="gradle">gradle</option>
- <option value="graphql">graphql</option>
- <option value="gql">gql</option>
- <option value="groovy">groovy</option>
- <option value="haml">haml</option>
- <option value="handlebars">handlebars</option>
- <option value="hbs">hbs</option>
- <option value="html.hbs">html.hbs</option>
- <option value="html.handlebars">html.handlebars</option>
- <option value="haskell">haskell</option>
- <option value="hs">hs</option>
- <option value="haxe">haxe</option>
- <option value="hx">hx</option>
- <option value="xml">xml</option>
- <option value="html">html</option>
- <option value="xhtml">xhtml</option>
- <option value="rss">rss</option>
- <option value="atom">atom</option>
- <option value="xjb">xjb</option>
- <option value="xsd">xsd</option>
- <option value="xsl">xsl</option>
- <option value="plist">plist</option>
- <option value="svg">svg</option>
- <option value="http">http</option>
- <option value="https">https</option>
- <option value="hy">hy</option>
- <option value="hylang">hylang</option>
- <option value="inform7">inform7</option>
- <option value="i7">i7</option>
- <option value="ini">ini</option>
- <option value="toml">toml</option>
- <option value="irpf90">irpf90</option>
- <option value="java">java</option>
- <option value="jsp">jsp</option>
- <option value="javascript">javascript</option>
- <option value="js">js</option>
- <option value="jsx">jsx</option>
- <option value="json">json</option>
- <option value="jsonc">jsonc</option>
- <option value="json5">json5</option>
- <option value="julia">julia</option>
- <option value="jl">jl</option>
- <option value="julia-repl">julia-repl</option>
- <option value="kotlin">kotlin</option>
- <option value="kt">kt</option>
- <option value="lasso">lasso</option>
- <option value="lassoscript">lassoscript</option>
- <option value="tex">tex</option>
- <option value="ldif">ldif</option>
- <option value="leaf">leaf</option>
- <option value="less">less</option>
- <option value="lisp">lisp</option>
- <option value="livecodeserver">livecodeserver</option>
- <option value="livescript">livescript</option>
- <option value="lua">lua</option>
- <option value="pluto">pluto</option>
- <option value="makefile">makefile</option>
- <option value="mk">mk</option>
- <option value="mak">mak</option>
- <option value="make">make</option>
- <option value="markdown">markdown</option>
- <option value="md">md</option>
- <option value="mkdown">mkdown</option>
- <option value="mkd">mkd</option>
- <option value="mathematica">mathematica</option>
- <option value="mma">mma</option>
- <option value="wl">wl</option>
- <option value="matlab">matlab</option>
- <option value="maxima">maxima</option>
- <option value="mel">mel</option>
- <option value="mercury">mercury</option>
- <option value="mips">mips</option>
- <option value="mipsasm">mipsasm</option>
- <option value="mizar">mizar</option>
- <option value="mojolicious">mojolicious</option>
- <option value="monkey">monkey</option>
- <option value="moonscript">moonscript</option>
- <option value="moon">moon</option>
- <option value="n1ql">n1ql</option>
- <option value="nginx">nginx</option>
- <option value="nginxconf">nginxconf</option>
- <option value="nim">nim</option>
- <option value="nimrod">nimrod</option>
- <option value="nix">nix</option>
- <option value="nsis">nsis</option>
- <option value="objectivec">objectivec</option>
- <option value="mm">mm</option>
- <option value="objc">objc</option>
- <option value="obj-c">obj-c</option>
- <option value="obj-c++">obj-c++</option>
- <option value="objective-c++">objective-c++</option>
- <option value="ocaml">ocaml</option>
- <option value="glsl">glsl</option>
- <option value="openscad">openscad</option>
- <option value="scad">scad</option>
- <option value="ruleslanguage">ruleslanguage</option>
- <option value="oxygene">oxygene</option>
- <option value="parser3">parser3</option>
- <option value="perl">perl</option>
- <option value="pl">pl</option>
- <option value="pm">pm</option>
- <option value="pf">pf</option>
- <option value="pf.conf">pf.conf</option>
- <option value="php">php</option>
- <option value="plaintext">plaintext</option>
- <option value="txt">txt</option>
- <option value="text">text</option>
- <option value="pony">pony</option>
- <option value="pgsql">pgsql</option>
- <option value="postgres">postgres</option>
- <option value="postgresql">postgresql</option>
- <option value="powershell">powershell</option>
- <option value="ps">ps</option>
- <option value="ps1">ps1</option>
- <option value="processing">processing</option>
- <option value="prolog">prolog</option>
- <option value="properties">properties</option>
- <option value="proto">proto</option>
- <option value="protobuf">protobuf</option>
- <option value="puppet">puppet</option>
- <option value="pp">pp</option>
- <option value="python">python</option>
- <option value="py">py</option>
- <option value="gyp">gyp</option>
- <option value="profile">profile</option>
- <option value="python-repl">python-repl</option>
- <option value="pycon">pycon</option>
- <option value="k">k</option>
- <option value="kdb">kdb</option>
- <option value="qml">qml</option>
- <option value="r">r</option>
- <option value="reasonml">reasonml</option>
- <option value="rib">rib</option>
- <option value="rsl">rsl</option>
- <option value="graph">graph</option>
- <option value="instances">instances</option>
- <option value="ruby">ruby</option>
- <option value="rb">rb</option>
- <option value="gemspec">gemspec</option>
- <option value="podspec">podspec</option>
- <option value="thor">thor</option>
- <option value="irb">irb</option>
- <option value="rust">rust</option>
- <option value="rs">rs</option>
- <option value="sas">sas</option>
- <option value="scala">scala</option>
- <option value="scheme">scheme</option>
- <option value="scilab">scilab</option>
- <option value="sci">sci</option>
- <option value="scss">scss</option>
- <option value="shell">shell</option>
- <option value="console">console</option>
- <option value="smali">smali</option>
- <option value="smalltalk">smalltalk</option>
- <option value="st">st</option>
- <option value="sml">sml</option>
- <option value="sql">sql</option>
- <option value="stan">stan</option>
- <option value="stanfuncs">stanfuncs</option>
- <option value="stata">stata</option>
- <option value="p21">p21</option>
- <option value="step">step</option>
- <option value="stp">stp</option>
- <option value="stylus">stylus</option>
- <option value="styl">styl</option>
- <option value="subunit">subunit</option>
- <option value="swift">swift</option>
- <option value="tcl">tcl</option>
- <option value="tk">tk</option>
- <option value="tap">tap</option>
- <option value="thrift">thrift</option>
- <option value="tp">tp</option>
- <option value="twig">twig</option>
- <option value="craftcms">craftcms</option>
- <option value="typescript">typescript</option>
- <option value="ts">ts</option>
- <option value="tsx">tsx</option>
- <option value="mts">mts</option>
- <option value="cts">cts</option>
- <option value="vala">vala</option>
- <option value="vbnet">vbnet</option>
- <option value="vb">vb</option>
- <option value="vbscript">vbscript</option>
- <option value="vbs">vbs</option>
- <option value="verilog">verilog</option>
- <option value="v">v</option>
- <option value="vhdl">vhdl</option>
- <option value="vim">vim</option>
- <option value="axapta">axapta</option>
- <option value="x++">x++</option>
- <option value="x86asm">x86asm</option>
- <option value="xl">xl</option>
- <option value="tao">tao</option>
- <option value="xquery">xquery</option>
- <option value="xpath">xpath</option>
- <option value="xq">xq</option>
- <option value="xqm">xqm</option>
- <option value="yml">yml</option>
- <option value="yaml">yaml</option>
- <option value="zephir">zephir</option>
- <option value="zep">zep</option>
+ <option value="">(auto)</option>
</select></div><br><input type="password" id="passphrase" autocomplete="off"
placeholder="passphrase (optional)"><small>&nbsp;encrypted with <a href="https://github.com/kj-sh604/mojicrypt"
target="_blank" rel="noopener noreferrer">mojicrypt</a></small>
diff --git a/src/main.js b/src/main.js
index 1f1b755..e0a9600 100644
--- a/src/main.js
+++ b/src/main.js
@@ -1,5 +1,223 @@
const FORM_STATE_KEY = "kj-clipboard-form-state-v1";
+const HIGHLIGHTJS_LANGUAGES = [
+ "1c",
+ "abnf",
+ "accesslog",
+ "actionscript",
+ "ada",
+ "angelscript",
+ "apache",
+ "applescript",
+ "arcade",
+ "arduino",
+ "armasm",
+ "xml",
+ "asciidoc",
+ "aspectj",
+ "autohotkey",
+ "autoit",
+ "avrasm",
+ "awk",
+ "axapta",
+ "bash",
+ "basic",
+ "bnf",
+ "brainfuck",
+ "c",
+ "cal",
+ "capnproto",
+ "ceylon",
+ "clean",
+ "clojure",
+ "clojure-repl",
+ "cmake",
+ "coffeescript",
+ "coq",
+ "cos",
+ "cpp",
+ "crmsh",
+ "crystal",
+ "csharp",
+ "csp",
+ "css",
+ "d",
+ "markdown",
+ "dart",
+ "delphi",
+ "diff",
+ "django",
+ "dns",
+ "dockerfile",
+ "dos",
+ "dsconfig",
+ "dts",
+ "dust",
+ "ebnf",
+ "elixir",
+ "elm",
+ "ruby",
+ "erb",
+ "erlang-repl",
+ "erlang",
+ "excel",
+ "fix",
+ "flix",
+ "fortran",
+ "fsharp",
+ "gams",
+ "gauss",
+ "gcode",
+ "gherkin",
+ "glsl",
+ "gml",
+ "go",
+ "golo",
+ "gradle",
+ "graphql",
+ "groovy",
+ "haml",
+ "handlebars",
+ "haskell",
+ "haxe",
+ "hsp",
+ "http",
+ "hy",
+ "inform7",
+ "ini",
+ "irpf90",
+ "isbl",
+ "java",
+ "javascript",
+ "jboss-cli",
+ "json",
+ "julia",
+ "julia-repl",
+ "kotlin",
+ "lasso",
+ "latex",
+ "ldif",
+ "leaf",
+ "less",
+ "lisp",
+ "livecodeserver",
+ "livescript",
+ "llvm",
+ "lsl",
+ "lua",
+ "makefile",
+ "mathematica",
+ "matlab",
+ "maxima",
+ "mel",
+ "mercury",
+ "mipsasm",
+ "mizar",
+ "perl",
+ "mojolicious",
+ "monkey",
+ "moonscript",
+ "n1ql",
+ "nestedtext",
+ "nginx",
+ "nim",
+ "nix",
+ "node-repl",
+ "nsis",
+ "objectivec",
+ "ocaml",
+ "openscad",
+ "oxygene",
+ "parser3",
+ "pf",
+ "pgsql",
+ "php",
+ "php-template",
+ "plaintext",
+ "pony",
+ "powershell",
+ "processing",
+ "profile",
+ "prolog",
+ "properties",
+ "protobuf",
+ "puppet",
+ "purebasic",
+ "python",
+ "python-repl",
+ "q",
+ "qml",
+ "r",
+ "reasonml",
+ "rib",
+ "roboconf",
+ "routeros",
+ "rsl",
+ "ruleslanguage",
+ "rust",
+ "sas",
+ "scala",
+ "scheme",
+ "scilab",
+ "scss",
+ "shell",
+ "smali",
+ "smalltalk",
+ "sml",
+ "sqf",
+ "sql",
+ "stan",
+ "stata",
+ "step21",
+ "stylus",
+ "subunit",
+ "swift",
+ "taggerscript",
+ "yaml",
+ "tap",
+ "tcl",
+ "thrift",
+ "tp",
+ "twig",
+ "typescript",
+ "vala",
+ "vbnet",
+ "vbscript",
+ "vbscript-html",
+ "verilog",
+ "vhdl",
+ "vim",
+ "wasm",
+ "wren",
+ "x86asm",
+ "xl",
+ "xquery",
+ "zephir",
+];
+
+function syncLanguageOptions() {
+ const langSelect = document.getElementById("lang-select");
+ const selected = langSelect.value;
+
+ langSelect.innerHTML = "";
+
+ const autoOption = document.createElement("option");
+ autoOption.value = "";
+ autoOption.textContent = "(auto)";
+ langSelect.appendChild(autoOption);
+
+ for (const language of HIGHLIGHTJS_LANGUAGES) {
+ const option = document.createElement("option");
+ option.value = language;
+ option.textContent = language;
+ langSelect.appendChild(option);
+ }
+
+ if (selected && HIGHLIGHTJS_LANGUAGES.includes(selected)) {
+ langSelect.value = selected;
+ }
+}
+
function saveFormState() {
const state = {
content: document.getElementById("content").value,
@@ -110,6 +328,7 @@ function setStatus(msg) {
document.getElementById("status").textContent = msg;
}
+syncLanguageOptions();
restoreFormState();
document.getElementById("get-link-btn").addEventListener("click", createPaste);
diff --git a/src/server.py b/src/server.py
index bddee31..303c1bf 100644
--- a/src/server.py
+++ b/src/server.py
@@ -7,6 +7,7 @@
import http.server
import json
import ipaddress
+import queue
import re
import secrets
import signal
@@ -52,6 +53,14 @@ SQLITE_SYNCHRONOUS = "NORMAL" # OFF | NORMAL | FULL | EXTRA
# accept short bursts without immediately refusing tcp connections.
HTTP_REQUEST_QUEUE_SIZE = 64
+# single sqlite writer with queue + micro-batching reduces lock contention under load.
+WRITE_QUEUE_MAX_SIZE = 4096
+WRITE_QUEUE_WAIT_SECONDS = 8
+WRITE_QUEUE_ENQUEUE_TIMEOUT_SECONDS = 0.25
+WRITE_BATCH_SIZE = 32
+WRITE_BATCH_MAX_DELAY_MS = 12
+WRITE_WORKER_JOIN_TIMEOUT_SECONDS = 5
+
TRUST_PROXY = False
TRUSTED_PROXY_IPS = {"127.0.0.1", "::1"}
# hsts off by default to avoid breaking plain-http setups.
@@ -63,324 +72,194 @@ ALLOWED_LANGUAGES = {
"abnf",
"accesslog",
"actionscript",
- "as",
"ada",
"angelscript",
- "asc",
"apache",
- "apacheconf",
"applescript",
- "osascript",
"arcade",
"arduino",
- "ino",
"armasm",
- "arm",
+ "xml",
"asciidoc",
- "adoc",
"aspectj",
"autohotkey",
"autoit",
"avrasm",
"awk",
- "mawk",
- "nawk",
- "gawk",
- "ballerina",
- "bal",
+ "axapta",
"bash",
- "sh",
- "zsh",
"basic",
"bnf",
"brainfuck",
- "bf",
"c",
- "h",
- "csharp",
- "cpp",
- "hpp",
- "cc",
- "hh",
- "c++",
- "h++",
- "cxx",
- "hxx",
"cal",
- "cos",
- "cls",
"capnproto",
- "capnp",
+ "ceylon",
+ "clean",
"clojure",
- "clj",
+ "clojure-repl",
"cmake",
- "cmake.in",
"coffeescript",
- "coffee",
- "cson",
- "iced",
"coq",
+ "cos",
+ "cpp",
"crmsh",
- "crm",
- "pcmk",
"crystal",
- "cr",
+ "csharp",
"csp",
"css",
"d",
+ "markdown",
"dart",
- "dpr",
- "dfm",
- "pas",
- "pascal",
+ "delphi",
"diff",
- "patch",
"django",
- "jinja",
"dns",
- "zone",
- "bind",
"dockerfile",
- "docker",
"dos",
- "bat",
- "cmd",
"dsconfig",
"dts",
"dust",
- "dst",
"ebnf",
"elixir",
"elm",
+ "ruby",
+ "erb",
+ "erlang-repl",
"erlang",
- "erl",
"excel",
- "xls",
- "xlsx",
- "extempore",
- "xtlang",
- "xtm",
- "fsharp",
- "fs",
- "fsx",
- "fsi",
- "fsscript",
"fix",
+ "flix",
+ "fsharp",
"fortran",
- "f90",
- "f95",
"gcode",
- "nc",
"gams",
- "gms",
"gauss",
- "gss",
"gherkin",
+ "glsl",
+ "gml",
"go",
- "golang",
"golo",
- "gololang",
"gradle",
"graphql",
- "gql",
"groovy",
"haml",
"handlebars",
- "hbs",
- "html.hbs",
- "html.handlebars",
"haskell",
- "hs",
"haxe",
- "hx",
- "xml",
- "html",
- "xhtml",
- "rss",
- "atom",
- "xjb",
- "xsd",
- "xsl",
- "plist",
- "svg",
+ "hsp",
"http",
- "https",
"hy",
- "hylang",
"inform7",
- "i7",
"ini",
- "toml",
"irpf90",
+ "isbl",
"java",
- "jsp",
"javascript",
- "js",
- "jsx",
+ "jboss-cli",
"json",
- "jsonc",
- "json5",
"julia",
- "jl",
"julia-repl",
"kotlin",
- "kt",
"lasso",
- "lassoscript",
- "tex",
+ "latex",
"ldif",
"leaf",
"less",
"lisp",
"livecodeserver",
"livescript",
+ "llvm",
+ "lsl",
"lua",
- "pluto",
"makefile",
- "mk",
- "mak",
- "make",
- "markdown",
- "md",
- "mkdown",
- "mkd",
"mathematica",
- "mma",
- "wl",
"matlab",
"maxima",
"mel",
"mercury",
- "mips",
"mipsasm",
"mizar",
+ "perl",
"mojolicious",
"monkey",
"moonscript",
- "moon",
"n1ql",
+ "nestedtext",
"nginx",
- "nginxconf",
"nim",
- "nimrod",
"nix",
+ "node-repl",
"nsis",
"objectivec",
- "mm",
- "objc",
- "obj-c",
- "obj-c++",
- "objective-c++",
"ocaml",
- "glsl",
"openscad",
- "scad",
- "ruleslanguage",
"oxygene",
"parser3",
- "perl",
- "pl",
- "pm",
"pf",
- "pf.conf",
+ "pgsql",
"php",
+ "php-template",
"plaintext",
- "txt",
- "text",
"pony",
- "pgsql",
- "postgres",
- "postgresql",
"powershell",
- "ps",
- "ps1",
"processing",
"prolog",
"properties",
- "proto",
"protobuf",
"puppet",
- "pp",
+ "purebasic",
"python",
- "py",
- "gyp",
"profile",
"python-repl",
- "pycon",
- "k",
- "kdb",
+ "q",
"qml",
"r",
"reasonml",
"rib",
+ "roboconf",
+ "routeros",
"rsl",
- "graph",
- "instances",
- "ruby",
- "rb",
- "gemspec",
- "podspec",
- "thor",
- "irb",
+ "ruleslanguage",
"rust",
- "rs",
"sas",
"scala",
"scheme",
"scilab",
- "sci",
"scss",
"shell",
- "console",
"smali",
"smalltalk",
- "st",
"sml",
+ "sqf",
"sql",
"stan",
- "stanfuncs",
"stata",
- "p21",
- "step",
- "stp",
+ "step21",
"stylus",
- "styl",
"subunit",
"swift",
- "tcl",
- "tk",
+ "taggerscript",
+ "yaml",
"tap",
+ "tcl",
"thrift",
"tp",
"twig",
- "craftcms",
"typescript",
- "ts",
- "tsx",
- "mts",
- "cts",
"vala",
"vbnet",
- "vb",
"vbscript",
- "vbs",
+ "vbscript-html",
"verilog",
- "v",
"vhdl",
"vim",
- "axapta",
- "x++",
+ "wasm",
+ "wren",
"x86asm",
"xl",
- "tao",
"xquery",
- "xpath",
- "xq",
- "xqm",
- "yml",
- "yaml",
"zephir",
- "zep",
}
@@ -389,6 +268,10 @@ ALLOWED_LANGUAGES = {
_rate_lock = threading.Lock()
_rate_state = {}
+_write_queue = queue.Queue(maxsize=WRITE_QUEUE_MAX_SIZE)
+_write_worker_thread = None
+_write_worker_stop = threading.Event()
+
# database
@@ -408,6 +291,147 @@ def sqlite_retry_sleep(attempt):
time.sleep(min((delay_ms + jitter_ms) / 1000.0, 1.0))
+def build_write_job(content, language, is_code, is_encrypted):
+ return {
+ "content": content,
+ "language": language,
+ "is_code": int(is_code),
+ "is_encrypted": int(is_encrypted),
+ "created_at": int(time.time()),
+ "paste_id": None,
+ "error": None,
+ "done": threading.Event(),
+ }
+
+
+def execute_write_batch(conn, jobs):
+ for write_attempt in range(SQLITE_WRITE_RETRIES + 1):
+ try:
+ conn.execute("BEGIN IMMEDIATE")
+ for job in jobs:
+ for _ in range(5):
+ paste_id = generate_id()
+ try:
+ conn.execute(
+ "INSERT INTO pastes (id, content, language, is_code, is_encrypted, created_at) "
+ "VALUES (?, ?, ?, ?, ?, ?)",
+ (
+ paste_id,
+ job["content"],
+ job["language"],
+ job["is_code"],
+ job["is_encrypted"],
+ job["created_at"],
+ ),
+ )
+ job["paste_id"] = paste_id
+ break
+ except sqlite3.IntegrityError:
+ continue
+ if not job["paste_id"]:
+ raise RuntimeError("failed to generate unique paste id")
+
+ conn.commit()
+ return
+ except sqlite3.OperationalError as err:
+ try:
+ conn.rollback()
+ except sqlite3.DatabaseError:
+ pass
+
+ if is_sqlite_busy_error(err):
+ if write_attempt >= SQLITE_WRITE_RETRIES:
+ raise DatabaseBusyError("database is busy; retry shortly") from err
+ sqlite_retry_sleep(write_attempt)
+ continue
+
+ raise
+ except Exception:
+ try:
+ conn.rollback()
+ except sqlite3.DatabaseError:
+ pass
+ raise
+
+
+def flush_write_batch(conn, jobs):
+ try:
+ execute_write_batch(conn, jobs)
+ except Exception as err:
+ for job in jobs:
+ job["error"] = err
+ job["done"].set()
+ return
+
+ for job in jobs:
+ job["done"].set()
+
+
+def collect_write_batch(first_job):
+ jobs = [first_job]
+ deadline = time.monotonic() + (WRITE_BATCH_MAX_DELAY_MS / 1000.0)
+
+ while len(jobs) < WRITE_BATCH_SIZE:
+ remaining = deadline - time.monotonic()
+ if remaining <= 0:
+ break
+
+ try:
+ next_job = _write_queue.get(timeout=remaining)
+ except queue.Empty:
+ break
+
+ jobs.append(next_job)
+
+ return jobs
+
+
+def write_worker_loop():
+ conn = open_db()
+ try:
+ while True:
+ if _write_worker_stop.is_set() and _write_queue.empty():
+ break
+
+ try:
+ first_job = _write_queue.get(timeout=0.25)
+ except queue.Empty:
+ continue
+
+ jobs = collect_write_batch(first_job)
+ flush_write_batch(conn, jobs)
+ for _ in jobs:
+ _write_queue.task_done()
+ finally:
+ conn.close()
+
+
+def start_write_worker():
+ global _write_worker_thread
+
+ if _write_worker_thread and _write_worker_thread.is_alive():
+ return
+
+ _write_worker_stop.clear()
+ _write_worker_thread = threading.Thread(
+ target=write_worker_loop,
+ daemon=True,
+ name="sqlite-write-worker",
+ )
+ _write_worker_thread.start()
+
+
+def stop_write_worker():
+ global _write_worker_thread
+
+ if not _write_worker_thread:
+ return
+
+ _write_worker_stop.set()
+ _write_worker_thread.join(timeout=WRITE_WORKER_JOIN_TIMEOUT_SECONDS)
+ _write_worker_thread = None
+
+
def open_db():
conn = sqlite3.connect(
str(DB_PATH),
@@ -465,47 +489,25 @@ def is_valid_paste_id(paste_id):
def save_paste(content, language=None, is_code=False, is_encrypted=False):
"""store a paste in the database, return its id"""
- for write_attempt in range(SQLITE_WRITE_RETRIES + 1):
- conn = open_db()
- try:
- # reserve the write lock early to reduce lock thrash under bursty writes.
- conn.execute("BEGIN IMMEDIATE")
- for _ in range(5):
- paste_id = generate_id()
- try:
- conn.execute(
- "INSERT INTO pastes (id, content, language, is_code, is_encrypted, created_at) "
- "VALUES (?, ?, ?, ?, ?, ?)",
- (
- paste_id,
- content,
- language,
- int(is_code),
- int(is_encrypted),
- int(time.time()),
- ),
- )
- conn.commit()
- return paste_id
- except sqlite3.IntegrityError:
- continue
- conn.rollback()
- raise RuntimeError("failed to generate unique paste id")
- except sqlite3.OperationalError as err:
- try:
- conn.rollback()
- except sqlite3.DatabaseError:
- pass
- if is_sqlite_busy_error(err):
- if write_attempt >= SQLITE_WRITE_RETRIES:
- raise DatabaseBusyError("database is busy; retry shortly") from err
- sqlite_retry_sleep(write_attempt)
- continue
- raise
- finally:
- conn.close()
+ job = build_write_job(content, language, is_code, is_encrypted)
- raise DatabaseBusyError("database is busy; retry shortly")
+ try:
+ _write_queue.put(job, timeout=WRITE_QUEUE_ENQUEUE_TIMEOUT_SECONDS)
+ except queue.Full as err:
+ raise DatabaseBusyError("write queue is full; retry shortly") from err
+
+ if not job["done"].wait(WRITE_QUEUE_WAIT_SECONDS):
+ raise DatabaseBusyError("write queue timeout; retry shortly")
+
+ if job["error"]:
+ if isinstance(job["error"], Exception):
+ raise job["error"]
+ raise RuntimeError("write failed")
+
+ if not job["paste_id"]:
+ raise RuntimeError("write completed without paste id")
+
+ return job["paste_id"]
def get_paste(paste_id):
@@ -672,8 +674,8 @@ def paste_page(paste, csp_nonce):
highlight_css = ""
highlight_js = ""
if is_code:
- highlight_css = '<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/styles/vs2015.min.css">'
- highlight_js = f"""<script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/highlight.min.js"></script>
+ highlight_css = '<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.11.1/styles/vs2015.min.css">'
+ highlight_js = f"""<script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.11.1/highlight.min.js"></script>
<script{script_nonce_attr}>hljs.highlightAll();</script>"""
return f"""<!DOCTYPE html>
@@ -1222,6 +1224,7 @@ class ClipboardHTTPServer(http.server.ThreadingHTTPServer):
def main():
init_db()
+ start_write_worker()
print(f"kj-clipboard - listening on {BIND}:{PORT}")
server = ClipboardHTTPServer((BIND, PORT), ClipboardHandler)
@@ -1245,6 +1248,7 @@ def main():
except KeyboardInterrupt:
print("\nreceived keyboard interrupt, shutting down.")
finally:
+ stop_write_worker()
server.server_close()