diff options
| author | kj_sh604 | 2026-04-18 14:25:31 -0400 |
|---|---|---|
| committer | kj_sh604 | 2026-04-18 14:25:31 -0400 |
| commit | f4b6ffce91bcf9cd3c4ef85dd1a2b30b043c2d4d (patch) | |
| tree | 3630313212ce37e90311e5f1fd3bfac5c0d5a566 | |
| parent | abf56914d2e9b2658b29be96811930ceedac9aa7 (diff) | |
refactor: better concurrency and use base highlight options
| -rw-r--r-- | src/index.html | 324 | ||||
| -rw-r--r-- | src/main.js | 219 | ||||
| -rw-r--r-- | src/server.py | 430 |
3 files changed, 437 insertions, 536 deletions
diff --git a/src/index.html b/src/index.html index 8d89859..f5510a2 100644 --- a/src/index.html +++ b/src/index.html @@ -96,329 +96,7 @@ <div class="controls"><button id="get-link-btn" type="button" title="generate snippet link">generate link</button><span class="spacer"></span><label><input type="checkbox" id="is-code">is this code? </label><select id="lang-select"> - <option value="">auto-detect</option> - <option value="1c">1c</option> - <option value="abnf">abnf</option> - <option value="accesslog">accesslog</option> - <option value="actionscript">actionscript</option> - <option value="as">as</option> - <option value="ada">ada</option> - <option value="angelscript">angelscript</option> - <option value="asc">asc</option> - <option value="apache">apache</option> - <option value="apacheconf">apacheconf</option> - <option value="applescript">applescript</option> - <option value="osascript">osascript</option> - <option value="arcade">arcade</option> - <option value="arduino">arduino</option> - <option value="ino">ino</option> - <option value="armasm">armasm</option> - <option value="arm">arm</option> - <option value="asciidoc">asciidoc</option> - <option value="adoc">adoc</option> - <option value="aspectj">aspectj</option> - <option value="autohotkey">autohotkey</option> - <option value="autoit">autoit</option> - <option value="avrasm">avrasm</option> - <option value="awk">awk</option> - <option value="mawk">mawk</option> - <option value="nawk">nawk</option> - <option value="gawk">gawk</option> - <option value="ballerina">ballerina</option> - <option value="bal">bal</option> - <option value="bash">bash</option> - <option value="sh">sh</option> - <option value="zsh">zsh</option> - <option value="basic">basic</option> - <option value="bnf">bnf</option> - <option value="brainfuck">brainfuck</option> - <option value="bf">bf</option> - <option value="c">c</option> - <option value="h">h</option> - <option value="csharp">csharp</option> - <option value="cpp">cpp</option> - <option value="hpp">hpp</option> - <option value="cc">cc</option> - <option value="hh">hh</option> - <option value="c++">c++</option> - <option value="h++">h++</option> - <option value="cxx">cxx</option> - <option value="hxx">hxx</option> - <option value="cal">cal</option> - <option value="cos">cos</option> - <option value="cls">cls</option> - <option value="capnproto">capnproto</option> - <option value="capnp">capnp</option> - <option value="clojure">clojure</option> - <option value="clj">clj</option> - <option value="cmake">cmake</option> - <option value="cmake.in">cmake.in</option> - <option value="coffeescript">coffeescript</option> - <option value="coffee">coffee</option> - <option value="cson">cson</option> - <option value="iced">iced</option> - <option value="coq">coq</option> - <option value="crmsh">crmsh</option> - <option value="crm">crm</option> - <option value="pcmk">pcmk</option> - <option value="crystal">crystal</option> - <option value="cr">cr</option> - <option value="csp">csp</option> - <option value="css">css</option> - <option value="d">d</option> - <option value="dart">dart</option> - <option value="dpr">dpr</option> - <option value="dfm">dfm</option> - <option value="pas">pas</option> - <option value="pascal">pascal</option> - <option value="diff">diff</option> - <option value="patch">patch</option> - <option value="django">django</option> - <option value="jinja">jinja</option> - <option value="dns">dns</option> - <option value="zone">zone</option> - <option value="bind">bind</option> - <option value="dockerfile">dockerfile</option> - <option value="docker">docker</option> - <option value="dos">dos</option> - <option value="bat">bat</option> - <option value="cmd">cmd</option> - <option value="dsconfig">dsconfig</option> - <option value="dts">dts</option> - <option value="dust">dust</option> - <option value="dst">dst</option> - <option value="ebnf">ebnf</option> - <option value="elixir">elixir</option> - <option value="elm">elm</option> - <option value="erlang">erlang</option> - <option value="erl">erl</option> - <option value="excel">excel</option> - <option value="xls">xls</option> - <option value="xlsx">xlsx</option> - <option value="extempore">extempore</option> - <option value="xtlang">xtlang</option> - <option value="xtm">xtm</option> - <option value="fsharp">fsharp</option> - <option value="fs">fs</option> - <option value="fsx">fsx</option> - <option value="fsi">fsi</option> - <option value="fsscript">fsscript</option> - <option value="fix">fix</option> - <option value="fortran">fortran</option> - <option value="f90">f90</option> - <option value="f95">f95</option> - <option value="gcode">gcode</option> - <option value="nc">nc</option> - <option value="gams">gams</option> - <option value="gms">gms</option> - <option value="gauss">gauss</option> - <option value="gss">gss</option> - <option value="gherkin">gherkin</option> - <option value="go">go</option> - <option value="golang">golang</option> - <option value="golo">golo</option> - <option value="gololang">gololang</option> - <option value="gradle">gradle</option> - <option value="graphql">graphql</option> - <option value="gql">gql</option> - <option value="groovy">groovy</option> - <option value="haml">haml</option> - <option value="handlebars">handlebars</option> - <option value="hbs">hbs</option> - <option value="html.hbs">html.hbs</option> - <option value="html.handlebars">html.handlebars</option> - <option value="haskell">haskell</option> - <option value="hs">hs</option> - <option value="haxe">haxe</option> - <option value="hx">hx</option> - <option value="xml">xml</option> - <option value="html">html</option> - <option value="xhtml">xhtml</option> - <option value="rss">rss</option> - <option value="atom">atom</option> - <option value="xjb">xjb</option> - <option value="xsd">xsd</option> - <option value="xsl">xsl</option> - <option value="plist">plist</option> - <option value="svg">svg</option> - <option value="http">http</option> - <option value="https">https</option> - <option value="hy">hy</option> - <option value="hylang">hylang</option> - <option value="inform7">inform7</option> - <option value="i7">i7</option> - <option value="ini">ini</option> - <option value="toml">toml</option> - <option value="irpf90">irpf90</option> - <option value="java">java</option> - <option value="jsp">jsp</option> - <option value="javascript">javascript</option> - <option value="js">js</option> - <option value="jsx">jsx</option> - <option value="json">json</option> - <option value="jsonc">jsonc</option> - <option value="json5">json5</option> - <option value="julia">julia</option> - <option value="jl">jl</option> - <option value="julia-repl">julia-repl</option> - <option value="kotlin">kotlin</option> - <option value="kt">kt</option> - <option value="lasso">lasso</option> - <option value="lassoscript">lassoscript</option> - <option value="tex">tex</option> - <option value="ldif">ldif</option> - <option value="leaf">leaf</option> - <option value="less">less</option> - <option value="lisp">lisp</option> - <option value="livecodeserver">livecodeserver</option> - <option value="livescript">livescript</option> - <option value="lua">lua</option> - <option value="pluto">pluto</option> - <option value="makefile">makefile</option> - <option value="mk">mk</option> - <option value="mak">mak</option> - <option value="make">make</option> - <option value="markdown">markdown</option> - <option value="md">md</option> - <option value="mkdown">mkdown</option> - <option value="mkd">mkd</option> - <option value="mathematica">mathematica</option> - <option value="mma">mma</option> - <option value="wl">wl</option> - <option value="matlab">matlab</option> - <option value="maxima">maxima</option> - <option value="mel">mel</option> - <option value="mercury">mercury</option> - <option value="mips">mips</option> - <option value="mipsasm">mipsasm</option> - <option value="mizar">mizar</option> - <option value="mojolicious">mojolicious</option> - <option value="monkey">monkey</option> - <option value="moonscript">moonscript</option> - <option value="moon">moon</option> - <option value="n1ql">n1ql</option> - <option value="nginx">nginx</option> - <option value="nginxconf">nginxconf</option> - <option value="nim">nim</option> - <option value="nimrod">nimrod</option> - <option value="nix">nix</option> - <option value="nsis">nsis</option> - <option value="objectivec">objectivec</option> - <option value="mm">mm</option> - <option value="objc">objc</option> - <option value="obj-c">obj-c</option> - <option value="obj-c++">obj-c++</option> - <option value="objective-c++">objective-c++</option> - <option value="ocaml">ocaml</option> - <option value="glsl">glsl</option> - <option value="openscad">openscad</option> - <option value="scad">scad</option> - <option value="ruleslanguage">ruleslanguage</option> - <option value="oxygene">oxygene</option> - <option value="parser3">parser3</option> - <option value="perl">perl</option> - <option value="pl">pl</option> - <option value="pm">pm</option> - <option value="pf">pf</option> - <option value="pf.conf">pf.conf</option> - <option value="php">php</option> - <option value="plaintext">plaintext</option> - <option value="txt">txt</option> - <option value="text">text</option> - <option value="pony">pony</option> - <option value="pgsql">pgsql</option> - <option value="postgres">postgres</option> - <option value="postgresql">postgresql</option> - <option value="powershell">powershell</option> - <option value="ps">ps</option> - <option value="ps1">ps1</option> - <option value="processing">processing</option> - <option value="prolog">prolog</option> - <option value="properties">properties</option> - <option value="proto">proto</option> - <option value="protobuf">protobuf</option> - <option value="puppet">puppet</option> - <option value="pp">pp</option> - <option value="python">python</option> - <option value="py">py</option> - <option value="gyp">gyp</option> - <option value="profile">profile</option> - <option value="python-repl">python-repl</option> - <option value="pycon">pycon</option> - <option value="k">k</option> - <option value="kdb">kdb</option> - <option value="qml">qml</option> - <option value="r">r</option> - <option value="reasonml">reasonml</option> - <option value="rib">rib</option> - <option value="rsl">rsl</option> - <option value="graph">graph</option> - <option value="instances">instances</option> - <option value="ruby">ruby</option> - <option value="rb">rb</option> - <option value="gemspec">gemspec</option> - <option value="podspec">podspec</option> - <option value="thor">thor</option> - <option value="irb">irb</option> - <option value="rust">rust</option> - <option value="rs">rs</option> - <option value="sas">sas</option> - <option value="scala">scala</option> - <option value="scheme">scheme</option> - <option value="scilab">scilab</option> - <option value="sci">sci</option> - <option value="scss">scss</option> - <option value="shell">shell</option> - <option value="console">console</option> - <option value="smali">smali</option> - <option value="smalltalk">smalltalk</option> - <option value="st">st</option> - <option value="sml">sml</option> - <option value="sql">sql</option> - <option value="stan">stan</option> - <option value="stanfuncs">stanfuncs</option> - <option value="stata">stata</option> - <option value="p21">p21</option> - <option value="step">step</option> - <option value="stp">stp</option> - <option value="stylus">stylus</option> - <option value="styl">styl</option> - <option value="subunit">subunit</option> - <option value="swift">swift</option> - <option value="tcl">tcl</option> - <option value="tk">tk</option> - <option value="tap">tap</option> - <option value="thrift">thrift</option> - <option value="tp">tp</option> - <option value="twig">twig</option> - <option value="craftcms">craftcms</option> - <option value="typescript">typescript</option> - <option value="ts">ts</option> - <option value="tsx">tsx</option> - <option value="mts">mts</option> - <option value="cts">cts</option> - <option value="vala">vala</option> - <option value="vbnet">vbnet</option> - <option value="vb">vb</option> - <option value="vbscript">vbscript</option> - <option value="vbs">vbs</option> - <option value="verilog">verilog</option> - <option value="v">v</option> - <option value="vhdl">vhdl</option> - <option value="vim">vim</option> - <option value="axapta">axapta</option> - <option value="x++">x++</option> - <option value="x86asm">x86asm</option> - <option value="xl">xl</option> - <option value="tao">tao</option> - <option value="xquery">xquery</option> - <option value="xpath">xpath</option> - <option value="xq">xq</option> - <option value="xqm">xqm</option> - <option value="yml">yml</option> - <option value="yaml">yaml</option> - <option value="zephir">zephir</option> - <option value="zep">zep</option> + <option value="">(auto)</option> </select></div><br><input type="password" id="passphrase" autocomplete="off" placeholder="passphrase (optional)"><small> encrypted with <a href="https://github.com/kj-sh604/mojicrypt" target="_blank" rel="noopener noreferrer">mojicrypt</a></small> diff --git a/src/main.js b/src/main.js index 1f1b755..e0a9600 100644 --- a/src/main.js +++ b/src/main.js @@ -1,5 +1,223 @@ const FORM_STATE_KEY = "kj-clipboard-form-state-v1"; +const HIGHLIGHTJS_LANGUAGES = [ + "1c", + "abnf", + "accesslog", + "actionscript", + "ada", + "angelscript", + "apache", + "applescript", + "arcade", + "arduino", + "armasm", + "xml", + "asciidoc", + "aspectj", + "autohotkey", + "autoit", + "avrasm", + "awk", + "axapta", + "bash", + "basic", + "bnf", + "brainfuck", + "c", + "cal", + "capnproto", + "ceylon", + "clean", + "clojure", + "clojure-repl", + "cmake", + "coffeescript", + "coq", + "cos", + "cpp", + "crmsh", + "crystal", + "csharp", + "csp", + "css", + "d", + "markdown", + "dart", + "delphi", + "diff", + "django", + "dns", + "dockerfile", + "dos", + "dsconfig", + "dts", + "dust", + "ebnf", + "elixir", + "elm", + "ruby", + "erb", + "erlang-repl", + "erlang", + "excel", + "fix", + "flix", + "fortran", + "fsharp", + "gams", + "gauss", + "gcode", + "gherkin", + "glsl", + "gml", + "go", + "golo", + "gradle", + "graphql", + "groovy", + "haml", + "handlebars", + "haskell", + "haxe", + "hsp", + "http", + "hy", + "inform7", + "ini", + "irpf90", + "isbl", + "java", + "javascript", + "jboss-cli", + "json", + "julia", + "julia-repl", + "kotlin", + "lasso", + "latex", + "ldif", + "leaf", + "less", + "lisp", + "livecodeserver", + "livescript", + "llvm", + "lsl", + "lua", + "makefile", + "mathematica", + "matlab", + "maxima", + "mel", + "mercury", + "mipsasm", + "mizar", + "perl", + "mojolicious", + "monkey", + "moonscript", + "n1ql", + "nestedtext", + "nginx", + "nim", + "nix", + "node-repl", + "nsis", + "objectivec", + "ocaml", + "openscad", + "oxygene", + "parser3", + "pf", + "pgsql", + "php", + "php-template", + "plaintext", + "pony", + "powershell", + "processing", + "profile", + "prolog", + "properties", + "protobuf", + "puppet", + "purebasic", + "python", + "python-repl", + "q", + "qml", + "r", + "reasonml", + "rib", + "roboconf", + "routeros", + "rsl", + "ruleslanguage", + "rust", + "sas", + "scala", + "scheme", + "scilab", + "scss", + "shell", + "smali", + "smalltalk", + "sml", + "sqf", + "sql", + "stan", + "stata", + "step21", + "stylus", + "subunit", + "swift", + "taggerscript", + "yaml", + "tap", + "tcl", + "thrift", + "tp", + "twig", + "typescript", + "vala", + "vbnet", + "vbscript", + "vbscript-html", + "verilog", + "vhdl", + "vim", + "wasm", + "wren", + "x86asm", + "xl", + "xquery", + "zephir", +]; + +function syncLanguageOptions() { + const langSelect = document.getElementById("lang-select"); + const selected = langSelect.value; + + langSelect.innerHTML = ""; + + const autoOption = document.createElement("option"); + autoOption.value = ""; + autoOption.textContent = "(auto)"; + langSelect.appendChild(autoOption); + + for (const language of HIGHLIGHTJS_LANGUAGES) { + const option = document.createElement("option"); + option.value = language; + option.textContent = language; + langSelect.appendChild(option); + } + + if (selected && HIGHLIGHTJS_LANGUAGES.includes(selected)) { + langSelect.value = selected; + } +} + function saveFormState() { const state = { content: document.getElementById("content").value, @@ -110,6 +328,7 @@ function setStatus(msg) { document.getElementById("status").textContent = msg; } +syncLanguageOptions(); restoreFormState(); document.getElementById("get-link-btn").addEventListener("click", createPaste); diff --git a/src/server.py b/src/server.py index bddee31..303c1bf 100644 --- a/src/server.py +++ b/src/server.py @@ -7,6 +7,7 @@ import http.server import json import ipaddress +import queue import re import secrets import signal @@ -52,6 +53,14 @@ SQLITE_SYNCHRONOUS = "NORMAL" # OFF | NORMAL | FULL | EXTRA # accept short bursts without immediately refusing tcp connections. HTTP_REQUEST_QUEUE_SIZE = 64 +# single sqlite writer with queue + micro-batching reduces lock contention under load. +WRITE_QUEUE_MAX_SIZE = 4096 +WRITE_QUEUE_WAIT_SECONDS = 8 +WRITE_QUEUE_ENQUEUE_TIMEOUT_SECONDS = 0.25 +WRITE_BATCH_SIZE = 32 +WRITE_BATCH_MAX_DELAY_MS = 12 +WRITE_WORKER_JOIN_TIMEOUT_SECONDS = 5 + TRUST_PROXY = False TRUSTED_PROXY_IPS = {"127.0.0.1", "::1"} # hsts off by default to avoid breaking plain-http setups. @@ -63,324 +72,194 @@ ALLOWED_LANGUAGES = { "abnf", "accesslog", "actionscript", - "as", "ada", "angelscript", - "asc", "apache", - "apacheconf", "applescript", - "osascript", "arcade", "arduino", - "ino", "armasm", - "arm", + "xml", "asciidoc", - "adoc", "aspectj", "autohotkey", "autoit", "avrasm", "awk", - "mawk", - "nawk", - "gawk", - "ballerina", - "bal", + "axapta", "bash", - "sh", - "zsh", "basic", "bnf", "brainfuck", - "bf", "c", - "h", - "csharp", - "cpp", - "hpp", - "cc", - "hh", - "c++", - "h++", - "cxx", - "hxx", "cal", - "cos", - "cls", "capnproto", - "capnp", + "ceylon", + "clean", "clojure", - "clj", + "clojure-repl", "cmake", - "cmake.in", "coffeescript", - "coffee", - "cson", - "iced", "coq", + "cos", + "cpp", "crmsh", - "crm", - "pcmk", "crystal", - "cr", + "csharp", "csp", "css", "d", + "markdown", "dart", - "dpr", - "dfm", - "pas", - "pascal", + "delphi", "diff", - "patch", "django", - "jinja", "dns", - "zone", - "bind", "dockerfile", - "docker", "dos", - "bat", - "cmd", "dsconfig", "dts", "dust", - "dst", "ebnf", "elixir", "elm", + "ruby", + "erb", + "erlang-repl", "erlang", - "erl", "excel", - "xls", - "xlsx", - "extempore", - "xtlang", - "xtm", - "fsharp", - "fs", - "fsx", - "fsi", - "fsscript", "fix", + "flix", + "fsharp", "fortran", - "f90", - "f95", "gcode", - "nc", "gams", - "gms", "gauss", - "gss", "gherkin", + "glsl", + "gml", "go", - "golang", "golo", - "gololang", "gradle", "graphql", - "gql", "groovy", "haml", "handlebars", - "hbs", - "html.hbs", - "html.handlebars", "haskell", - "hs", "haxe", - "hx", - "xml", - "html", - "xhtml", - "rss", - "atom", - "xjb", - "xsd", - "xsl", - "plist", - "svg", + "hsp", "http", - "https", "hy", - "hylang", "inform7", - "i7", "ini", - "toml", "irpf90", + "isbl", "java", - "jsp", "javascript", - "js", - "jsx", + "jboss-cli", "json", - "jsonc", - "json5", "julia", - "jl", "julia-repl", "kotlin", - "kt", "lasso", - "lassoscript", - "tex", + "latex", "ldif", "leaf", "less", "lisp", "livecodeserver", "livescript", + "llvm", + "lsl", "lua", - "pluto", "makefile", - "mk", - "mak", - "make", - "markdown", - "md", - "mkdown", - "mkd", "mathematica", - "mma", - "wl", "matlab", "maxima", "mel", "mercury", - "mips", "mipsasm", "mizar", + "perl", "mojolicious", "monkey", "moonscript", - "moon", "n1ql", + "nestedtext", "nginx", - "nginxconf", "nim", - "nimrod", "nix", + "node-repl", "nsis", "objectivec", - "mm", - "objc", - "obj-c", - "obj-c++", - "objective-c++", "ocaml", - "glsl", "openscad", - "scad", - "ruleslanguage", "oxygene", "parser3", - "perl", - "pl", - "pm", "pf", - "pf.conf", + "pgsql", "php", + "php-template", "plaintext", - "txt", - "text", "pony", - "pgsql", - "postgres", - "postgresql", "powershell", - "ps", - "ps1", "processing", "prolog", "properties", - "proto", "protobuf", "puppet", - "pp", + "purebasic", "python", - "py", - "gyp", "profile", "python-repl", - "pycon", - "k", - "kdb", + "q", "qml", "r", "reasonml", "rib", + "roboconf", + "routeros", "rsl", - "graph", - "instances", - "ruby", - "rb", - "gemspec", - "podspec", - "thor", - "irb", + "ruleslanguage", "rust", - "rs", "sas", "scala", "scheme", "scilab", - "sci", "scss", "shell", - "console", "smali", "smalltalk", - "st", "sml", + "sqf", "sql", "stan", - "stanfuncs", "stata", - "p21", - "step", - "stp", + "step21", "stylus", - "styl", "subunit", "swift", - "tcl", - "tk", + "taggerscript", + "yaml", "tap", + "tcl", "thrift", "tp", "twig", - "craftcms", "typescript", - "ts", - "tsx", - "mts", - "cts", "vala", "vbnet", - "vb", "vbscript", - "vbs", + "vbscript-html", "verilog", - "v", "vhdl", "vim", - "axapta", - "x++", + "wasm", + "wren", "x86asm", "xl", - "tao", "xquery", - "xpath", - "xq", - "xqm", - "yml", - "yaml", "zephir", - "zep", } @@ -389,6 +268,10 @@ ALLOWED_LANGUAGES = { _rate_lock = threading.Lock() _rate_state = {} +_write_queue = queue.Queue(maxsize=WRITE_QUEUE_MAX_SIZE) +_write_worker_thread = None +_write_worker_stop = threading.Event() + # database @@ -408,6 +291,147 @@ def sqlite_retry_sleep(attempt): time.sleep(min((delay_ms + jitter_ms) / 1000.0, 1.0)) +def build_write_job(content, language, is_code, is_encrypted): + return { + "content": content, + "language": language, + "is_code": int(is_code), + "is_encrypted": int(is_encrypted), + "created_at": int(time.time()), + "paste_id": None, + "error": None, + "done": threading.Event(), + } + + +def execute_write_batch(conn, jobs): + for write_attempt in range(SQLITE_WRITE_RETRIES + 1): + try: + conn.execute("BEGIN IMMEDIATE") + for job in jobs: + for _ in range(5): + paste_id = generate_id() + try: + conn.execute( + "INSERT INTO pastes (id, content, language, is_code, is_encrypted, created_at) " + "VALUES (?, ?, ?, ?, ?, ?)", + ( + paste_id, + job["content"], + job["language"], + job["is_code"], + job["is_encrypted"], + job["created_at"], + ), + ) + job["paste_id"] = paste_id + break + except sqlite3.IntegrityError: + continue + if not job["paste_id"]: + raise RuntimeError("failed to generate unique paste id") + + conn.commit() + return + except sqlite3.OperationalError as err: + try: + conn.rollback() + except sqlite3.DatabaseError: + pass + + if is_sqlite_busy_error(err): + if write_attempt >= SQLITE_WRITE_RETRIES: + raise DatabaseBusyError("database is busy; retry shortly") from err + sqlite_retry_sleep(write_attempt) + continue + + raise + except Exception: + try: + conn.rollback() + except sqlite3.DatabaseError: + pass + raise + + +def flush_write_batch(conn, jobs): + try: + execute_write_batch(conn, jobs) + except Exception as err: + for job in jobs: + job["error"] = err + job["done"].set() + return + + for job in jobs: + job["done"].set() + + +def collect_write_batch(first_job): + jobs = [first_job] + deadline = time.monotonic() + (WRITE_BATCH_MAX_DELAY_MS / 1000.0) + + while len(jobs) < WRITE_BATCH_SIZE: + remaining = deadline - time.monotonic() + if remaining <= 0: + break + + try: + next_job = _write_queue.get(timeout=remaining) + except queue.Empty: + break + + jobs.append(next_job) + + return jobs + + +def write_worker_loop(): + conn = open_db() + try: + while True: + if _write_worker_stop.is_set() and _write_queue.empty(): + break + + try: + first_job = _write_queue.get(timeout=0.25) + except queue.Empty: + continue + + jobs = collect_write_batch(first_job) + flush_write_batch(conn, jobs) + for _ in jobs: + _write_queue.task_done() + finally: + conn.close() + + +def start_write_worker(): + global _write_worker_thread + + if _write_worker_thread and _write_worker_thread.is_alive(): + return + + _write_worker_stop.clear() + _write_worker_thread = threading.Thread( + target=write_worker_loop, + daemon=True, + name="sqlite-write-worker", + ) + _write_worker_thread.start() + + +def stop_write_worker(): + global _write_worker_thread + + if not _write_worker_thread: + return + + _write_worker_stop.set() + _write_worker_thread.join(timeout=WRITE_WORKER_JOIN_TIMEOUT_SECONDS) + _write_worker_thread = None + + def open_db(): conn = sqlite3.connect( str(DB_PATH), @@ -465,47 +489,25 @@ def is_valid_paste_id(paste_id): def save_paste(content, language=None, is_code=False, is_encrypted=False): """store a paste in the database, return its id""" - for write_attempt in range(SQLITE_WRITE_RETRIES + 1): - conn = open_db() - try: - # reserve the write lock early to reduce lock thrash under bursty writes. - conn.execute("BEGIN IMMEDIATE") - for _ in range(5): - paste_id = generate_id() - try: - conn.execute( - "INSERT INTO pastes (id, content, language, is_code, is_encrypted, created_at) " - "VALUES (?, ?, ?, ?, ?, ?)", - ( - paste_id, - content, - language, - int(is_code), - int(is_encrypted), - int(time.time()), - ), - ) - conn.commit() - return paste_id - except sqlite3.IntegrityError: - continue - conn.rollback() - raise RuntimeError("failed to generate unique paste id") - except sqlite3.OperationalError as err: - try: - conn.rollback() - except sqlite3.DatabaseError: - pass - if is_sqlite_busy_error(err): - if write_attempt >= SQLITE_WRITE_RETRIES: - raise DatabaseBusyError("database is busy; retry shortly") from err - sqlite_retry_sleep(write_attempt) - continue - raise - finally: - conn.close() + job = build_write_job(content, language, is_code, is_encrypted) - raise DatabaseBusyError("database is busy; retry shortly") + try: + _write_queue.put(job, timeout=WRITE_QUEUE_ENQUEUE_TIMEOUT_SECONDS) + except queue.Full as err: + raise DatabaseBusyError("write queue is full; retry shortly") from err + + if not job["done"].wait(WRITE_QUEUE_WAIT_SECONDS): + raise DatabaseBusyError("write queue timeout; retry shortly") + + if job["error"]: + if isinstance(job["error"], Exception): + raise job["error"] + raise RuntimeError("write failed") + + if not job["paste_id"]: + raise RuntimeError("write completed without paste id") + + return job["paste_id"] def get_paste(paste_id): @@ -672,8 +674,8 @@ def paste_page(paste, csp_nonce): highlight_css = "" highlight_js = "" if is_code: - highlight_css = '<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/styles/vs2015.min.css">' - highlight_js = f"""<script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/highlight.min.js"></script> + highlight_css = '<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.11.1/styles/vs2015.min.css">' + highlight_js = f"""<script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.11.1/highlight.min.js"></script> <script{script_nonce_attr}>hljs.highlightAll();</script>""" return f"""<!DOCTYPE html> @@ -1222,6 +1224,7 @@ class ClipboardHTTPServer(http.server.ThreadingHTTPServer): def main(): init_db() + start_write_worker() print(f"kj-clipboard - listening on {BIND}:{PORT}") server = ClipboardHTTPServer((BIND, PORT), ClipboardHandler) @@ -1245,6 +1248,7 @@ def main(): except KeyboardInterrupt: print("\nreceived keyboard interrupt, shutting down.") finally: + stop_write_worker() server.server_close() |
