article: circle-rasterization

.gitignore

@@ -1,4 +1,4 @@
-**/__pycache__/*
+**/__pycache__/
 html/
 ./.*/
 [articles/**/.static/]

articles/circle-rasterization/page.mmd

@@ -0,0 +1,84 @@
+Title:  Circle Rasterization
+Brief:  Investigation on fast grid-aligned circle rasterization.
+Date:   1737757212
+Tags:   Programming, Optimization, C
+CSS:    /style.css
+
+![](/articles/circle-rasterization/circles.webp)
+
+Currently drastically overthinking anything related to dream Minecraft-like game of mine,
+and today it was all about chunk loading. Particularly, ideal way to infer which chunks
+should be loaded based on distance to the viewer, instead of typical direct grid.
+
+For that circle rasterization is needed. I came up with following pieces of code, one reusable macro,
+and others are meant to be directly copy pasted where needed:
+
+Macro:
+```c
+/* Emits `x` and `y` for every intersecting cell */
+/* We snap position to the nearest corner, which means there's no aliasing */
+/* It works great for integer radii */
+#define m_iter_circle_pixels(p_center_x, p_center_y, p_radius) \
+    for (float y = (p_center_y + ceilf(p_radius)) - 1; y > (p_center_y - ceilf(p_radius)) - 1; --y) \
+        for (float x = p_center_x - ceilf(sqrtf(p_radius * p_radius - (y - p_center_y + (y <= p_center_y)) * (y - p_center_y + (y <= p_center_y)))); \
+                   x < p_center_x + ceilf(sqrtf(p_radius * p_radius - (y - p_center_y + (y <= p_center_y)) * (y - p_center_y + (y <= p_center_y)))); ++x)
+```
+
+Floating point based one:
+```c
+float const rs = state->r * state->r;
+float const cr = ceilf(state->r);
+for (float iy = -cr; iy <= cr - 1; ++iy) {
+    float const dx = ceilf(sqrtf(rs - (iy + (iy <= 0)) * (iy + (iy <= 0))));
+    for (float ix = -dx; ix < dx; ++ix) {
+        /* iy and ix are floating point offsets from (0, 0) */
+    }
+}
+```
+
+Integer math based one:
+```c
+/* Neat shorthand making integer based loops drastically faster */
+static int32_t ceil_sqrt(int32_t const n) {
+    int32_t res = 1;
+    #pragma clang loop unroll_count(8)
+    while(res * res < n)
+        res++;
+    return res;
+}
+
+/* This one beats the float in raw performance, but might scale worse at increasing radii, assuming sqrt is a hardware intrinsic with known worst time */
+int32_t const rsi = (int32_t)state->r * (int32_t)state->r;
+for (int32_t iy = -(int32_t)state->r; iy <= (int32_t)state->r - 1; ++iy) {
+    int32_t const dx = ceil_sqrt(rsi - (iy + (iy <= 0)) * (iy + (iy <= 0)));
+    for (int32_t ix = -dx; ix < dx; ++ix) {
+        /* iy and ix are integer offsets from (0, 0) */
+    }
+}
+```
+
+Integer math based with accumulated ceil(sqrt()), the fastest I could come up with:
+```c
+int32_t const rsi = (int32_t)state->r * (int32_t)state->r;
+int32_t acc = 1;
+for (int32_t iy = (int32_t)state->r - 1; iy >= 0; --iy) {
+    while (acc * acc < rsi - iy * iy) acc++;
+    for (int32_t ix = -acc; ix < acc; ++ix) {
+        /* lower portion */
+        x = (float)ix;
+        y = (float)iy;
+        /* upper portion */
+        x = (float)ix;
+        y = (float)-iy - 1;
+    }
+}
+```
+
+Note that they assume center point at coordinate origin, quadrant symmetry and whole number radii.
+
+Benchmarks:
+```
+Profile 'float'         on average took: 0.001537s, worst case: 0.003272s, sample count: 277
+Profile 'int32_t'       on average took: 0.000726s, worst case: 0.002293s, sample count: 277
+Profile 'int32_t acc'   on average took: 0.000650s, worst case: 0.001732s, sample count: 277
+```

articles/fast-quad-rotation/page.mmd

@@ -0,0 +1,62 @@
+Title:  Fast Quad Rotation
+Brief:  A better way to rotate quads around their centers.
+Date:   1722126213
+Tags:   Programming, Optimization, C
+CSS:    /style.css
+
+A similar in essence trick to [by pi rotation](/articles/vector-pi-rotation.html), but with delta calculated
+for some corner which is reused later with negation and coordinate swap.
+
+Additionally `cos(a) = sqrt(1 - sin(a) ^ 2)` is used to reuse the result of sin(a),
+with `fast_sqrt()` for good measure.
+
+### Code ###
+```c
+/* http://www.azillionmonkeys.com/qed/sqroot.html */
+static inline float fast_sqrt(float x)
+{
+    union {
+        float f;
+        uint32_t u;
+    } pun = {.f = x};
+
+    pun.u += 127 << 23;
+    pun.u >>= 1;
+
+    return pun.f;
+}
+
+/* instead of calculating cosf again, - use sinf result */
+static inline t_fvec2 fast_cossine(float a) {
+    const float s = sinf(a);
+    return (t_fvec2){
+        .x = fast_sqrt(1.0f - s * s) *
+            (a >= (float)M_PI_2 && a < (float)(M_PI + M_PI_2) ? -1 : 1),
+        .y = s
+    };
+}
+
+/* final vertex calculation */
+const t_fvec2 t = fast_cossine(sprite.rotation + (float)M_PI_4);
+
+/* scaling by `M_SQRT1_2` is there to retain the quad size (Pythagorean stuffs). */
+const t_fvec2 d = {
+    .x = t.x * sprite.rect.w * (float)M_SQRT1_2,
+    .y = t.y * sprite.rect.h * (float)M_SQRT1_2,
+};
+
+const t_fvec2 c = frect_center(sprite.rect);
+
+/* upper-left */
+const t_fvec2 v0 = { c.x - d.x, c.y - d.y };
+
+/* bottom-left */
+const t_fvec2 v1 = { c.x - d.y, c.y + d.x };
+
+/* bottom-right */
+const t_fvec2 v2 = { c.x + d.x, c.y + d.y };
+
+/* upper-right */
+const t_fvec2 v3 = { c.x + d.y, c.y - d.x };
+
+```

articles/firefox-wasm-tail-call-benchmark/page.mmd

@@ -1,7 +1,7 @@
 Title:  Testing Firefox Wasm Tail Call
 Brief:  Or why assumptions are not always correct.
 Date:   1708076705
-Tags:   Wasm, Interpreters
+Tags:   Optimization, Wasm, Interpreters
 CSS:    /style.css
 
 ### Lore ###

articles/hand-opt-simplex-2d/page.mmd

@@ -1,7 +1,7 @@
 Title:  Hand Optimized Simplex 2D
 Brief:  Results of messing around with moving and hoisting stuff around.
 Date:   1688995095
-Tags:   Programming, GLSL, OpenGL
+Tags:   Programming, GLSL, OpenGL, Optimization
 CSS:    /style.css
 
 ![](/articles/hand-opt-simplex-2d/noise.png)

articles/links-of-the-open-world-programmer/page.mmd

@@ -0,0 +1,108 @@
+Title:  Links of the Open World Programmer
+Brief:  Various links to articles and technical documentation I've found along the way of making my engines, mostly centered around OpenGL.
+Date:   1714393506
+Tags:   Programming, OpenGL, GLSL, Generation
+CSS:    /style.css
+
+## optimization articles
+- [Various instancing hacks and methods, with benchmarks](https://solhsa.com/instancing.html)
+- [Buffered memory can have drastic differences in read speed](https://community.intel.com/t5/Developing-Games-on-Intel/glMapBuffer-reading-mapped-memory-is-very-slow/td-p/1059726)
+- [Selecting device is problematic with opengl, but there are some hacky ways](https://stackoverflow.com/questions/68469954/how-to-choose-specific-gpu-when-create-opengl-context)
+- [For linux environments it's a lot messier](https://bbs.archlinux.org/viewtopic.php?id=266456)
+- [About vertex caching in instanced draw](http://eelpi.gotdns.org/papers/fast_vert_cache_opt.html)
+- [Two level indexing](https://stackoverflow.com/questions/11148567/rendering-meshes-with-multiple-indices)
+- [OpenGL Insights](https://openglinsights.com/)
+- [Da Pipeline](https://fgiesen.wordpress.com/2011/07/09/a-trip-through-the-graphics-pipeline-2011-index/)
+- [Apple guide on GLES](https://developer.apple.com/library/archive/documentation/3DDrawing/Conceptual/OpenGLES_ProgrammingGuide/Introduction/Introduction.html)
+- [About GPU cache](https://www.rastergrid.com/blog/gpu-tech/2021/01/understanding-gpu-caches/)
+- [Nvidia guide](https://docs.nvidia.com/drive/drive_os_5.1.6.1L/nvvib_docs/index.html#page/DRIVE_OS_Linux_SDK_Development_Guide/Graphics/graphics_opengl.html)
+- [Optimizing Triangle Strips for Fast Rendering](http://www.cs.umd.edu/gvil/papers/av_ts.pdf)
+- [TEGRA specific GLES2 guide](https://docs.nvidia.com/drive/drive_os_5.1.6.1L/nvvib_docs/DRIVE_OS_Linux_SDK_Development_Guide/baggage/tegra_gles2_performance.pdf)
+- [List of old NVidia GLSL pragmas](http://www.icare3d.org/news_articles/nvidia_glsl_compiling_options.html)
+- [Radeon 9XXX series optimization guide](https://people.freedesktop.org/~mareko/radeon-9700-opengl-programming-and-optimization-guide.pdf)
+- [GLSL optimizations](https://www.khronos.org/opengl/wiki/GLSL_Optimizations)
+- [IPhone 3D Programming book](https://www.oreilly.com/library/view/iphone-3d-programming/9781449388133/bk01-toc.html)
+- [Article about cache utilization tips and techniques](https://johnnysswlab.com/make-your-programs-run-faster-by-better-using-the-data-cache/)
+- [Pixel Buffer Object](http://www.songho.ca/opengl/gl_pbo.html)
+- [Screen quads](https://stackoverflow.com/questions/2588875/whats-the-best-way-to-draw-a-fullscreen-quad-in-opengl-3-2)
+- [Performance problems with framebuffer swaps](https://stackoverflow.com/questions/10729352/framebuffer-fbo-render-to-texture-is-very-slow-using-opengl-es-2-0-on-android)
+- [Matrices inside textures](https://stackoverflow.com/questions/29672810/efficient-way-to-manage-matrices-within-a-graphic-application-using-texture-buff)
+- [OpenGL Insights - Asynchronous Buffer Transfers](https://zbook.org/read/448e5c_opengl-insights-university-of-pennsylvania.html)
+- [Scene rendering techniques presentation](https://on-demand.gputechconf.com/gtc/2014/presentations/S4379-opengl-44-scene-rendering-techniques.pdf)
+- [High-performance extension galore](http://behindthepixels.io/assets/files/High-performance,%20Low-Overhead%20Rendering%20with%20OpenGL%20and%20Vulkan%20-%20Edward%20Liu.pdf)
+- [NVidia example on shader based occlusion culling](https://github.com/nvpro-samples/gl_occlusion_culling)
+- [Packing](http://smt565.blogspot.com/2011/04/bit-packing-depth-and-normals.html)
+- [Thread on texture DMA when transferring](https://community.khronos.org/t/texture-performance/49104)
+- [More recent DMA thread](https://community.khronos.org/t/direct-memory-access-in-opengl/108312/22)
+- [Mesa driven GLSL optimizer, might be relevant for devices with poor optimizing compilers](https://github.com/aras-p/glsl-optimizer)
+- [In-depth OpenGL feature overview with hardware support listed](https://www.g-truc.net/doc/Effective%20OpenGL.pdf)
+- [Z-order curve to increase locality of multidimensional data](https://en.wikipedia.org/wiki/Z-order_curve#Texture_mapping)
+- [Thread group locality](https://developer.nvidia.com/blog/optimizing-compute-shaders-for-l2-locality-using-thread-group-id-swizzling/)
+- [Shader-db to prove instruction counts](https://blogs.igalia.com/apinheiro/2015/09/optimizing-shader-assembly-instruction-on-mesa-using-shader-db/)
+- [Texture cache](https://computergraphics.stackexchange.com/questions/357/is-using-many-texture-maps-bad-for-caching)
+- [Hierarchical Z map occlusion culling](https://www.rastergrid.com/blog/2010/10/hierarchical-z-map-based-occlusion-culling/)
+- [Reducing driver overhead](https://gdcvault.com/play/1020791/)
+- [Persistent mapping](https://www.khronos.org/opengl/wiki/Buffer_Object#Persistent_mapping)
+- [Post transform cache friendly way of rendering regular grids](http://www.ludicon.com/castano/blog/2009/02/optimal-grid-rendering/)
+- [Discussion on above](https://community.khronos.org/t/optimize-grid-rendering-for-post-t-l-cache/72272/9)
+- [Vertex optimization on modern GPUs](https://www.tugraz.at/fileadmin/user_upload/Institute/ICG/Images/team_steinberger/Pipelines/HPG-2018_shading_rate-authorversion.opt.pdf)
+- [General SIMD usage and techniques](https://repository.dl.itc.u-tokyo.ac.jp/record/48871/files/A32992.pdf)
+- [Fragment friendly circle meshing](http://www.humus.name/index.php?page=News&ID=228)
+- [Occlusion culling for terrain](https://www.researchgate.net/publication/248358913_Voxel_Column_Culling_Occlusion_Culling_For_Large_Terrain_Models)
+- [Billboard quad transformation optimization](https://gamedev.stackexchange.com/questions/201963/efficient-calculation-of-billboard-sprite-transformations)
+- [NVidia bindless extensions](https://developer.download.nvidia.com/opengl/tutorials/bindless_graphics.pdf)
+- [hacksoflife blog, full of good things](http://hacksoflife.blogspot.com/search/label/OpenGL)
+
+## technical stuff
+- [Determinism between opengl vendors](https://stackoverflow.com/questions/7922526/opengl-deterministic-rendering-between-gpu-vendor)
+- [Early fragment test, my beloved](https://www.khronos.org/opengl/wiki/Early_Fragment_Test)
+- [Shape digitalization](https://tug.org/docs/hobby/hobby-thesis.pdf)
+- [Line and circle rasterization](http://www.sunshine2k.de/coding/java/Bresenham/RasterisingLinesCircles.pdf)
+- [Occlusion culling of Vintage Story](https://github.com/tyronx/occlusionculling)
+- [Minecraft work on cave occlusion, in 2 parts](https://tomcc.github.io/2014/08/31/visibility-1.html)
+- [Order independent blending technique](https://jcgt.org/published/0002/02/09/)
+- [High performance voxel engine](https://nickmcd.me/2021/04/04/high-performance-voxel-engine/)
+- [Monotone meshing](https://blackflux.wordpress.com/tag/monotone-meshing/)
+- [Capsule collision detection](https://wickedengine.net/2020/04/26/capsule-collision-detection/)
+- [Forsyth vertex cache optimization](https://tomforsyth1000.github.io/papers/fast_vert_cache_opt.html)
+- [Depth buffer based lighting](https://www.researchgate.net/publication/320616607_Eye-Dome_Lighting_a_non-photorealistic_shading_technique)
+- [Computational Geometry in C (Second Edition)](http://www.science.smith.edu/~jorourke/books/compgeom.html)
+- [OpenGL FAQ](https://www.opengl.org/archives/resources/faq/technical/)
+- [SGI BSP FAQ](https://web.archive.org/web/20010614072959/http://reality.sgi.com/bspfaq/)
+
+## generational stuff
+- [Domain warping](https://iquilezles.org/articles/warp/)
+- [Portable and fast Perlin noise in legacy GLSL](https://arxiv.org/abs/1204.1461)
+- [Evaluation of GPU noise hashing solutions](https://jcgt.org/published/0009/03/02/paper.pdf)
+- [SHISHUA](https://espadrine.github.io/blog/posts/shishua-the-fastest-prng-in-the-world.html)
+- [Generalized lattice noise](https://www.codeproject.com/Articles/785084/A-generic-lattice-noise-algorithm-an-evolution-of)
+- [Procedural hydrology](https://nickmcd.me/2020/04/15/procedural-hydrology/)
+- [Tectonics](https://nickmcd.me/2020/12/03/clustered-convection-for-simulating-plate-tectonics/)
+- [Approximation of heightmaps](https://www.cs.cmu.edu/~garland/scape/scape.pdf)
+
+## notable extensions
+- [Vertex array locking](https://registry.khronos.org/OpenGL/extensions/EXT/EXT_compiled_vertex_array.txt)
+- [Packed pixels](https://people.freedesktop.org/~marcheu/extensions/EXT/packed_pixels.html)
+- [Framebuffer fetch](https://registry.khronos.org/OpenGL/extensions/EXT/EXT_shader_framebuffer_fetch.txt)
+- [Integer textures](https://registry.khronos.org/OpenGL/extensions/EXT/EXT_texture_integer.txt)
+- [Texture swizzle](https://registry.khronos.org/OpenGL/extensions/EXT/EXT_texture_swizzle.txt)
+- [Shader binaries](https://registry.khronos.org/OpenGL/extensions/ARB/ARB_get_program_binary.txt)
+- [Internal format query](https://registry.khronos.org/OpenGL/extensions/ARB/ARB_internalformat_query2.txt)
+- [Direct state access](https://registry.khronos.org/OpenGL/extensions/EXT/EXT_direct_state_access.txt)
+- [Texture view](https://registry.khronos.org/OpenGL/extensions/EXT/EXT_texture_view.txt)
+- [No error](https://registry.khronos.org/OpenGL/extensions/KHR/KHR_no_error.txt)
+- [Trinary min and max](https://registry.khronos.org/OpenGL/extensions/AMD/AMD_shader_trinary_minmax.txt)
+- [NV occlusion query, with partitioning, without locking, potentially with less overdraw](https://registry.khronos.org/OpenGL/extensions/NV/NV_conditional_render.txt)
+- [ES2 compatibility, can be used to query precision of floats](https://registry.khronos.org/OpenGL/extensions/ARB/ARB_ES2_compatibility.txt)
+- [Pipeline stats](https://registry.khronos.org/OpenGL/extensions/ARB/ARB_pipeline_statistics_query.txt)
+- [Parallel shader compile](https://registry.khronos.org/OpenGL/extensions/ARB/ARB_parallel_shader_compile.txt)
+- [Shader inter group communication](https://registry.khronos.org/OpenGL/extensions/ARB/ARB_shader_ballot.txt)
+- [Granular buffer memory control](https://registry.khronos.org/OpenGL/extensions/ARB/ARB_sparse_buffer.txt)
+- [Window pos](https://people.freedesktop.org/~marcheu/extensions/ARB/window_pos.html)
+- [No perspective interpolation for screen aligned geometry](https://registry.khronos.org/OpenGL/extensions/NV/NV_shader_noperspective_interpolation.txt)
+
+## data representations
+- [Efficient varying-length integers](https://john-millikin.com/vu128-efficient-variable-length-integers)
+- [Awesome article on hashtables](https://thenumb.at/Hashtables/)
+- [Crit-bit trees](https://cr.yp.to/critbit.html)
+- [QP tries](https://dotat.at/prog/qp/README.html)

articles/neocities-cache/page.mmd

@@ -1,7 +1,7 @@
 Title:  Cached Neocities Uploads
 Brief:  Making uploading of directories to Neocities less painful.
 Date:   1707585916
-Tags:   Bash
+Tags:   Programming, Bash, Script
 CSS:    /style.css
 
 Quick and dirty Bash-based sha256sum checksum solution to create stamps for later checking and rejection.

articles/oscillators/make

@@ -1,7 +1,7 @@
-#!/bin/sh
+#!/usr/bin/env bash
 
 cd $(dirname "$0")
 mkdir -p ./.dynamic
 mkdir -p ./.temp
-gcc -Wno-unused-result -Wno-incompatible-pointer-types waveforms.c ../../tools/gifenc/gifenc.c -I../../tools -O2 -o ./.temp/waveforms
+$CC -Wno-unused-result -Wno-incompatible-pointer-types waveforms.c ../../tools/gifenc/gifenc.c -I../../tools -O2 -o ./.temp/waveforms
 ./.temp/waveforms

articles/oscillators/page.mmd

@@ -51,12 +51,16 @@ struct sqrtwave {
     uint32_t u;
   } v;
 } init_sqrtwave(float frequency, float phase, float amplitude) {
-  struct sqrtwave r;
+    struct sqrtwave r;
-  r.w = init_sinewave(frequency, phase, 1.f);
+  union {
-  v.f = r.w.s;
+    float f;
-  a.f = amplitude;
+    uint32_t u;
-  r.v.u = (a.u & 0x7fffffff) | (v.u & 0x80000000);
+  } v, a;
-  return r;
+    r.w = init_sinewave(frequency, phase, 1.f);
+    v.f = r.w.s;
+    a.f = amplitude;
+    r.v.u = (a.u & 0x7fffffff) | (v.u & 0x80000000);
+    return r;
 }
 
 /* Use floating point bit representation to infer sign, all other bits are set to amplitude */

articles/rodata-lookup-caching/page.mmd

@@ -0,0 +1,90 @@
+Title:  Lookup Caching by .rodata Section String Inference
+Brief:  Rather hacky, but working way of string key lookup acceleration.
+Date:   1722127090
+Tags:   Programming, Optimization, C, Linux
+CSS:    /style.css
+
+While working on our immediate no-state engine, the need for texture lookup optimization arose.
+API is designed in a way where every single pushed triangle means resolution of texture by path.
+
+My insane mind came to such optimization then: detect whether given path pointer is in .rodata and if so, -
+just lookup by hash of the pointer itself, not whole varying-size string. Constant time and all that.
+
+For that I ended up writing a limited ELF parsing routine that expects `/proc/self/exe`.
+Virtual address space randomization was tricky until I realized that
+`getauxval(AT_ENTRY) - ehdr.e_entry` could be used to get the base process address.
+
+After the section bounds are known, - it's as simple as checking `vm_start >= ptr && ptr < vm_end`.
+
+### Code ###
+```c
+/* code is fully self-contained, feel free to use it :) */
+
+#include <fcntl.h>
+#include <unistd.h>
+#include <sys/auxv.h>
+#include <elf.h>
+#include <linux/limits.h>
+
+#include <stdbool.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+bool infer_elf_section_bounds(const char *const restrict name,
+                              const char **restrict vm_start,
+                              const char **restrict vm_end)
+{
+    bool result = false;
+    char buf[PATH_MAX];
+    ssize_t l = readlink("/proc/self/exe", buf, PATH_MAX);
+    if (l == -1)
+        goto ERR_CANT_READLINK;
+    buf[l] = 0; /* readlink() doesn't write a terminator */
+
+    int elf = open(buf, O_RDONLY);
+    if (elf == -1)
+        goto ERR_CANT_OPEN_SELF;
+
+    /* elf header */
+    Elf64_Ehdr ehdr;
+    read(elf, &ehdr, sizeof ehdr);
+    if (ehdr.e_ident[EI_MAG0] != ELFMAG0 ||
+            ehdr.e_ident[EI_MAG1] != ELFMAG1 ||
+            ehdr.e_ident[EI_MAG2] != ELFMAG2 ||
+            ehdr.e_ident[EI_MAG3] != ELFMAG3)
+        goto ERR_NOT_ELF;
+
+    /* section header string table */
+    Elf64_Shdr shstrdr;
+    lseek(elf, ehdr.e_shoff + ehdr.e_shstrndx * sizeof (Elf64_Shdr), SEEK_SET);
+    read(elf, &shstrdr, sizeof shstrdr);
+    char *sh = malloc(shstrdr.sh_size);
+    lseek(elf, shstrdr.sh_offset, SEEK_SET);
+    read(elf, sh, shstrdr.sh_size);
+
+    /* walk sections searching for needed name */
+    lseek(elf, ehdr.e_shoff, SEEK_SET);
+    for (size_t s = 0; s < ehdr.e_shnum; ++s) {
+        Elf64_Shdr shdr;
+        read(elf, &shdr, sizeof shdr);
+
+        if (strcmp(&sh[shdr.sh_name], name) == 0) {
+            result = true;
+            *vm_start = getauxval(AT_ENTRY) - ehdr.e_entry + (char *)shdr.sh_addr;
+            *vm_end   = getauxval(AT_ENTRY) - ehdr.e_entry + (char *)shdr.sh_addr + shdr.sh_size;
+            break;
+        }
+    }
+
+    free(sh);
+
+ERR_NOT_ELF:
+    close(elf);
+
+ERR_CANT_OPEN_SELF:
+ERR_CANT_READLINK:
+    return result;
+}
+
+```

articles/tiny-elf/page.mmd

@@ -1,7 +1,7 @@
 Title:  Slim Summer Elf
 Brief:  Making of minimal x86 (Linux) ELF executable.
 Date:   1684666702
-Tags:   Programming, Linux, C
+Tags:   Programming, Linux, C, Bash, Linker, Low-level
 CSS:    /style.css
 
 Code below was composed for [4mb-jam](https://itch.io/jam/4mb-jam-2023) which I didn't finish.

articles/toponym-extractor/page.mmd

@@ -0,0 +1,36 @@
+Title:  Geonames Toponym Extractor Utility
+Brief:  Simple script for extracting ASCII toponym fields from geonames datasets
+Date:   1713683410
+Tags:   Python, Script, Programming
+CSS:    /style.css
+
+[Link to code](https://codeberg.org/veclavtalica/geonames-extractor)
+
+Small script I used for extracting data for machine learning endeavors.
+
+Usage:
+```
+dataset feature_class [feature_code] [--dirty] [--filter=mask]
+```
+
+From this invokation ...
+```
+./extractor.py datasets/UA.txt P PPL --filter=0123456789\"\'-\` > UA-prep.txt
+```
+
+... it produces a newline separated list of relevant toponyms of particular kind, such as:
+```
+Katerynivka
+Vaniushkyne
+Svistuny
+Sopych
+Shilova Balka
+```
+
+`--filter=` option is there so that aplhabet size could be reduced for learning purposes,
+as there are usually quite a lot of symbols that are only found few times,
+which produces poor balancing.
+
+`--dirty` option reduces cases such as `Maydan (Ispas)` and `CHAYKA-Transmitter, Ring Mast 4` to `Maydan` and `CHAYKA-Transmitter`.
+
+Duplicates are also removed.

browse.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+
+set +e
+
+printf "%s" "Enter URL: "
+read URL
+
+articles=$(curl -s "$URL/articles.txt")
+articles=$(echo -n "$articles" | python3 -c "import sys; from urllib.parse import unquote; print(unquote(sys.stdin.read()));")
+
+while :
+do
+    article=$(./tools/widgets/list_selector.py --desc="Select an article:" --result="line" -- $articles)
+    if [ -z "$article" ]; then
+        break
+    fi
+    curl -s "$URL/markdown/$article.md" | pager
+done

compile.sh

@@ -1,7 +1,11 @@
-#!/bin/sh
+#!/usr/bin/env bash
 
 set +e
 
+# Settings:
+# =========
+export CC=cc
+
 mkdir -p ./html/articles
 
 ./tools/main_page_generator.py ./articles | ./tools/mmd/build/multimarkdown > ./html/index.html
@@ -9,7 +13,7 @@ mkdir -p ./html/articles
 for d in ./articles/*/; do
   if [ -d "$d" ]; then
     if test -f "$d/make"; then
-      "$d/make"
+      ("$d/make")
     fi
     if test -d "$d/.dynamic"; then
       mkdir -p "./html/articles/$(basename -- $d)"
@@ -19,8 +23,24 @@ for d in ./articles/*/; do
       mkdir -p "./html/articles/$(basename -- $d)"
       cp -r "$d/.static/." "./html/articles/$(basename -- $d)/"
     fi
-    ./tools/article_wrapper.py "$d/page.mmd" https://mjestecko.neocities.org | ./tools/mmd/build/multimarkdown > "./html/articles/$(basename -- $d).html"
+    ./tools/article_wrapper.py "$d/page.mmd" | ./tools/mmd/build/multimarkdown > "./html/articles/$(basename -- $d).html"
   fi
 done
 
-./tools/feed_generator.py ./articles/ https://mjestecko.neocities.org > ./html/feed.xml
+mkdir -p "./html/tags/"
+./tools/tag_listing_generator.py ./articles/ ./html/ | ./tools/mmd/build/multimarkdown > "./html/tags.html"
+
+for f in ./html/tags/*.html; do
+  echo $(cat "$f" | ./tools/mmd/build/multimarkdown) > "$f"
+done
+
+./tools/feed_generator.py ./articles/ > ./html/feed.xml
+
+./tools/plaintext_article_listing_generator.py ./articles/ > ./html/articles.txt
+
+mkdir -p "./html/markdown/"
+for d in ./articles/*/; do
+  if [ -d "$d" ]; then
+    cp "$d/page.mmd" "./html/markdown/$(basename $d).md"
+  fi
+done

config.py

@@ -0,0 +1,45 @@
+from random import choice
+
+## Title of the blog
+## Used for default first navbar entry to "/" root.
+## 
+title = "mjestečko"
+
+## Final hosting address, used in RSS feed absolute links as well as previews.
+##
+address = "https://mjestecko.neocities.org"
+
+## Shows on top of every page providing navigation.
+## Every entry forms a <li><a> child element of <nav>,
+## where each dictionary pair forms an attached xml property.
+## 
+navbar = [
+    ("source", { "href": "https://git.poto.cafe/veclavtalica/mjestecko" }),
+    ("rss", { "href": "/feed.xml" }),
+    ("about", { "href": "/articles/mjestečko.html" }),
+    ("tracks", { "href":  "https://modarchive.org/index.php?request=view_artist_modules&query=96070" }),
+    ("mastodon", { "href":  "https://poto.cafe/@veclavtalica", "rel": "me" }),
+]
+
+## Optional description that will be shown on top of the main page.
+## Could be plain text or callable with no parameters.
+## 
+description = lambda: f"Personal blog of one {choice(adjectives)} Veclav Talica."
+adjectives = ["*wild*", "**wacky**", "very humble", "**most serious**"]
+
+## Optional link to logo image that will appear on top of the main page.
+##
+logo = "/logo.png"
+
+## Language specifier, used in RSS feed.
+##
+language = "en"
+
+## Port that is used to listed to remote git push signals.
+##
+webhook_port = 14032
+
+## Something that only git hosting and your server should know.
+## See: https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Authorization
+##
+webhook_auth = "Basic you-secure-credentials"

cook_tracks.sh

@@ -1,9 +0,0 @@
-#!/usr/bin/bash
-
-set +e
-shopt -s extglob
-
-for f in $1/!(*AutoSave*).xm
-do
-  gzip -c $f | base64 --wrap=0 - | tr -d '\n' > $2/$(basename -- $f).txt
-done

html/style.css

@@ -1,102 +1,115 @@
-@media (prefers-color-scheme: dark) {
+ @media (prefers-color-scheme: dark) {
   :root {
     --bg-color: #111;
     --fg-color: #fff;
+    --hover-color: #e0e0e0;
     --link-color: dodgerblue;
+    --nav-border-color: #bbbbdd;
   }
 }
 
-* {
+ @media only screen and (max-device-width: 480px) {
+  article, .container {
+    width: 100%;
+  }
+  html {
+    font-size: 13px;
+  }
+}
+
+ * {
   color: var(--fg-color);
   text-decoration: none;
 }
-
+ a {
-a {
   color: var(--link-color);
 }
 
-html {
+ html {
-     font-size: 20px;
+  font-size: 20px;
-     font-family: "Charter","Georgia",'Times New Roman',serif;
+  font-family: "Charter", "Georgia", "Times New Roman", serif;
 }
+
  body {
-     background-color: var(--bg-color);
+  background-color: var(--bg-color);
-     line-height: 1.3;
+  line-height: 1.3;
 }
  article, .container {
-     max-width: 800px;
+  max-width: 800px;
-     margin: auto;
+  margin: auto;
 }
  h1,h2,h3,h4 {
-     font-family: arial, sans;
+  font-family: arial, sans;
 }
  h2 {
-     text-align: center;
+  text-align: center;
-     margin-bottom: 1em;
+  margin-bottom: 1em;
 }
  h3 {
-     font-family: arial, sans;
+  font-family: arial, sans;
-     margin: 1.3em 0 0.9em;
+  margin: 1.3em 0 0.9em;
 }
  b {
-     font-family: arial, sans;
+  font-family: arial, sans;
-     font-weight: normal;
+  font-weight: normal;
-     font-size: 0.8em;
+  font-size: 0.8em;
 }
  pre, p {
-     margin: 0.9em 0px 0.9em;
+  margin: 0.9em 0px 0.9em;
 }
  pre, code {
-     background-color: var(--bg-color);
+  background-color: var(--bg-color);
-     line-height: normal;
+  line-height: normal;
-     overflow: auto;
+  overflow: auto;
-     font-size: 0.8em;
+  font-size: 0.8em;
 }
  pre {
-     padding: 0.5em 0.5em;
+  padding: 0.5em 0.5em;
-     border-radius: 4px;
+  border-radius: 4px;
 }
  nav ul {
-     padding-left: 0;
+  padding-left: 0;
-     border-radius: 4px;
+  border-radius: 4px;
-     list-style: none;
+  list-style: none;
-     display: flex;
+  display: flex;
-     flex-direction: row;
+  flex-direction: row;
-     flex-wrap: nowrap;
+  flex-wrap: nowrap;
-     font-size: 0.8em;
+  font-size: 0.8em;
-     background-color: var(--bg-color);
+  background-color: var(--bg-color);
-     border: 1px solid #bbbbdd;
+  border: 1px solid var(--nav-border-color);
+  overflow: scroll;
 }
  nav li:first-child {
-     border-left: none;
+  border-left: none;
 }
  nav li {
-     border-right: 1px solid #bbbbdd;
+  border-right: 1px solid var(--nav-border-color);
-     border-left: 1px solid #ffffff;
+  border-left: 1px solid var(--fg-color);
 }
  nav a {
-     padding: 0.4em 1em;
+  padding: 0.4em 1em;
-     display: inline-block;
+  display: inline-block;
-     text-decoration: none;
+  text-decoration: none;
-     font-family: arial, sans;
+  font-family: arial, sans;
 }
  nav a:hover {
-     background-color: #e0e0e0;
+  background-color: var(--hover-color);
 }
  nav a:link {
-     color: var(--link-color);
+  color: var(--link-color);
 }
  nav a:visited {
-     color: var(--link-color);
+  color: var(--link-color);
 }
  nav a.here {
-     background-color: #e0e0e0;
+  background-color: var(--hover-color);
 }
- @media only screen and (max-device-width: 480px) {
+
-     article, .container {
+ img {
-         width: 100%;
+  max-width: 100%;
-    }
+  height: auto;
-     html {
+  width: auto;
-         font-size: 13px;
+  margin-left: auto;
-    }
+  margin-right: auto;
+  display: block;
 }

local_host.sh

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+python3 -m http.server --directory ./html/ & xdg-open http://0.0.0.0:8000/

new-article.sh

@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+
+set +e
+
+printf "%s" "Enter title: "
+read title
+
+printf "%s" "Enter directory name: "
+read directory
+
+if [ -d "./articles/$directory/" ]; then
+    echo "Directory already exists, aborted."
+    exit
+fi
+
+printf "%s" "Enter brief: "
+read brief
+
+printf "%s" "Enter tags: "
+read tags
+
+mkdir "./articles/$directory/"
+
+mask=$(cat <<-format
+Title:  %s
+Brief:  %s
+Date:   %s
+Tags:   %s
+CSS:    /style.css
+
+format
+)
+date=$(date +%s)
+
+printf "$mask" "$title" "$brief" "$date" "$tags" \
+    > "./articles/$directory/page.mmd"
+
+if which xdg-open &> /dev/null; then
+    xdg-open "./articles/$directory/page.mmd"
+fi

prepare.sh

@@ -1,9 +1,9 @@
-#!/bin/sh
+#!/usr/bin/env bash
 
-git submodule init
+set +e
-git submodule update
+
-cd tools/mmd
+git submodule update --init --recursive
-make release
+git-lfs fetch
-cd build
+
-make
+(cd tools/mmd && make release)
-cd ../..
+(cd tools/mmd/build && make)

remote_host.sh

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+python3 -m http.server --directory ./html/ & python3 ./tools/git_webhook.py

tools/article_utils.py

@@ -1,8 +1,10 @@
 import time, subprocess
+from os import walk, path
+import urllib.parse
 
 def the_line_after_metadata(lines: []) -> int:
     i = 0
-    while lines[i].strip():
+    while i < len(lines) and lines[i].strip():
         i += 1
     return i
 
@@ -18,9 +20,24 @@ def parse_metadata(filepath: str) -> {}:
         if key == "Date":
             result["Date"] = time.gmtime(int(val))
         elif key == "Tags":
-            result["Tags"] = val.split(",")
+            result["Tags"] = [v.strip() for v in val.split(",")]
-        else:
+        elif val:
             result[key] = val
         result["Last Edit"] = time.gmtime(int(subprocess.getoutput(r"stat -c %Y " + filepath)))
 
     return result
+
+def parse_article_directory(directory: str) -> {}:
+    articles = {}
+    for root, dirs, _ in walk(directory):
+        for d in dirs:
+            metadata = parse_metadata(path.abspath(root + '/' + d + "/page.mmd"))
+            article = urllib.parse.quote(d)
+            articles[article] = {
+                "metadata": metadata
+            }
+        break
+    return articles
+
+def sort_titles_by_date(articles: {}) -> []:
+    return sorted(articles.keys(), key=lambda a: articles[a]["metadata"].get("Date", time.gmtime(0)), reverse=True)

tools/article_wrapper.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 
 # todo: Show related git history of a file?
 
@@ -7,16 +7,15 @@ import time, urllib.parse, re
 import os.path as path
 
 from article_utils import the_line_after_metadata, parse_metadata
-from page_shares import wrap_page, MONTHS
+from page_builder import wrap_page
+from date_descriptions import MONTHS
+
+import config
 
 if len(argv) <= 1:
     print("No file was supplied")
     exit(-1)
 
-if len(argv) <= 2:
-    print("No address was supplied")
-    exit(-1)
-
 with open(argv[1], "r") as f:
     content = f.readlines()
     i = the_line_after_metadata(content)
@@ -24,7 +23,7 @@ with open(argv[1], "r") as f:
 metadata = parse_metadata(argv[1])
 directory = path.split(path.dirname(path.abspath(argv[1])))[-1]
 
-title = metadata.get("Title", "Oopsie, somebody forgot to name the article!")
+title = metadata["Title"]
 article_head = "\n# " + title + "\n"
 
 brief = metadata.get("Brief")
@@ -41,22 +40,25 @@ if not last_edit is None:
         last_edit.tm_mday != date.tm_mday or last_edit.tm_year != date.tm_year:
             article_head += f"-- Edited: *{MONTHS[last_edit.tm_mon]} {last_edit.tm_mday}, {last_edit.tm_year} UTC*\n\n"
 
-# todo: Hyperlinks to appropriate tag pages.
 tags = metadata.get("Tags")
 if tags:
-    article_head += f"""-- Tags: *{",".join(tags)}*\n\n"""
+    tag_links = []
+    for tag in tags:
+        tag_links.append(f"[{tag}](/tags/{urllib.parse.quote(tag.lower())}.html)")
+    article_head += f"""-- Tags: *{", ".join(tag_links)}*\n\n"""
 
 article_head += "---\n\n"
 
 header = f"""HTML header: <meta property="og:title" content="{title} on mjestečko"></meta>
     <meta property="og:type" content="article"></meta>
-    <meta property="og:url" content="{argv[2]}/articles/{urllib.parse.quote(directory)}.html"></meta>
+    <meta property="og:url" content="{config.address}/articles/{urllib.parse.quote(directory)}.html"></meta>
+    <meta name="viewport" content="width=device-width, initial-scale=1">
 """
 if not brief is None:
     header += f"""<meta property="og:description" content="{brief}"></meta>\n"""
 
 front_image = re.compile(r"!\[.*\]\((.+?)\)", re.DOTALL).search(''.join(content[i:]))
 if not front_image is None:
-    header += f"""<meta property="og:image" content="{argv[2]}/{urllib.parse.quote(front_image.group(1))}"></meta>\n"""
+    header += f"""<meta property="og:image" content="{config.address}/{urllib.parse.quote(front_image.group(1))}"></meta>\n"""
 
 print(header + ''.join(content[:i]) + wrap_page(article_head + ''.join(content[i:])))

tools/config.py

@@ -0,0 +1 @@
+../config.py

tools/date_descriptions.py

@@ -0,0 +1,24 @@
+MONTHS = {
+    1: "January",
+    2: "February",
+    3: "March",
+    4: "April",
+    5: "May",
+    6: "June",
+    7: "July",
+    8: "August",
+    9: "September",
+    10: "October",
+    11: "November",
+    12: "December"
+}
+
+WEEKDAYS = {
+    0: "Monday",
+    1: "Tuesday",
+    2: "Wednesday",
+    3: "Thursday",
+    4: "Friday",
+    5: "Saturday",
+    6: "Sunday"
+}

tools/feed_generator.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 
 from sys import argv, exit
 from os import walk, path
@@ -7,28 +7,23 @@ from textwrap import indent
 import time, urllib.parse, re, subprocess
 
 from article_utils import parse_metadata
-from page_shares import ADJECTIVES
 from rfc822 import stringify_date
 
+import config
+
 if len(argv) <= 1:
     print("No directory was supplied")
     exit(-1)
 
-if len(argv) <= 2:
-    print("No address was supplied")
-    exit(-1)
-
 seed()
 
-address = argv[2]
-
 # todo: Find the latest pubDate
 feed = f"""<rss version="2.0">
     <channel>
-        <title>mjestečko</title>
+        <title>{config.title}</title>
-        <link>{address}</link>
+        <link>{config.address}</link>
-        <description>Personal blog of one {choice(ADJECTIVES)} Veclav Talica</description>
+        <description>{config.description() if callable(config.description) else config.description}</description>
-        <language>en</language>
+        <language>{config.language}</language>
         <lastBuildDate>{stringify_date(time.gmtime(int(time.time())))}</lastBuildDate>
 """
 
@@ -54,7 +49,7 @@ for root, dirs, _ in walk(argv[1]):
                 f"""        <pubDate>{stringify_date(metadata["Date"])}</pubDate>\n"""
         feed += (
             f"""        <guid>/articles/{d}</guid>\n"""
-            f"""        <link>{address}/articles/{urllib.parse.quote(d)}</link>\n"""
+            f"""        <link>{config.address}/articles/{urllib.parse.quote(d)}</link>\n"""
             "    </item>\n"
         )
     break

tools/git_webhook.py

@@ -0,0 +1,56 @@
+#!/usr/bin/env python3
+
+from http.client import parse_headers
+from http.server import BaseHTTPRequestHandler
+from http.server import HTTPServer
+
+import subprocess
+
+import config
+
+## Simple way to automatically pull and recompile on a remote server.
+## Run this from the root directory.
+##
+## Currently supports:
+##  - Gitea (Via GET method).
+##
+
+
+class HttpHandler(BaseHTTPRequestHandler):
+    def do_GET(self):
+        got_gitea_push_event = False
+        got_auth = False
+
+        for header in self.headers:
+            match [header, self.headers[header]]:
+                case ["X-Gitea-Event", "push"]:
+                    got_gitea_push_event = True
+
+                case ["Authorization", config.webhook_auth]:
+                    got_auth = True
+
+
+        if not got_gitea_push_event or not got_auth:
+            self.send_response(400)
+            return
+
+        # todo: This way of doing it blocks both parties. Not ideal.
+        self.send_response(200)
+
+        subprocess.run(["git", "pull"])
+        subprocess.run(["./compile.sh"])
+
+        print("Pulled and recompiled.")
+
+
+def run(server_class=HTTPServer, handler_class=HttpHandler):
+    server_address = ('', config.webhook_port)
+    httpd = server_class(server_address, handler_class)
+    try:
+        httpd.serve_forever()
+    except KeyboardInterrupt:
+        httpd.server_close()
+
+
+if __name__ == "__main__":
+    run()

tools/main_page_generator.py

@@ -1,12 +1,14 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 
 from sys import argv, exit
-from os import walk, path
+from random import seed
-from random import choice, seed
+import time
-import time, urllib.parse
 
-from article_utils import parse_metadata
+from article_utils import parse_article_directory, sort_titles_by_date
-from page_shares import wrap_page, ADJECTIVES, MONTHS
+from page_builder import wrap_page
+from date_descriptions import MONTHS
+
+import config
 
 if len(argv) <= 1:
     print("No directory was supplied")
@@ -14,28 +16,23 @@ if len(argv) <= 1:
 
 seed()
 
-page_metadata = """Title:  mjestečko
+page_metadata = f"""Title:  {config.title}
 CSS:    /style.css
+HTML header:    <meta name="viewport" content="width=device-width, initial-scale=1">
 
 """
 
-page = f"""Personal blog of one {choice(ADJECTIVES)} Veclav Talica.
+page = f"""![]({config.logo})
+
+{config.description() if callable(config.description) else config.description}
 
 ### Articles ###
 
 """
 
-artciles = {}
+artciles = parse_article_directory(argv[1])
-for root, dirs, _ in walk(argv[1]):
-    for d in dirs:
-        metadata = parse_metadata(path.abspath(root + '/' + d + "/page.mmd"))
-        article = urllib.parse.quote(d)
-        artciles[article] = {
-            "metadata": metadata
-        }
-    break
 
-for title in sorted(artciles.keys(), key=lambda a: artciles[a]["metadata"].get("Date", time.gmtime(0)), reverse=True):
+for title in sort_titles_by_date(artciles):
     article = artciles[title]
     metadata = article["metadata"]
     page += (

tools/page_builder.py

@@ -0,0 +1,43 @@
+import config
+
+_navbar = config.navbar.copy()
+_navbar.insert(0, ('<strong>' + config.title + '</strong>', {"href": "/"}))
+_navbar.append(("tags", {"href": "/tags.html"}))
+
+_navbar_lis = '\n'.join(f"""<li><a {' '.join(
+        f'{p}="{v}"' for p, v in e[1].items())}>{e[0]}</a></li>"""
+    for e in _navbar)
+
+_head = f"""
+
+<div class="container">
+<nav>
+<ul>
+{_navbar_lis}
+</ul>
+</nav>
+
+"""
+
+_footer = """
+<footer>
+<a href="#top">^ Return</a>
+</footer>
+
+"""
+
+_tail = """
+
+</div>
+
+"""
+
+def mixin_tag(content: str, tag: str) -> str:
+    return f"""<{tag}>
+
+{content}</{tag}>
+
+"""
+
+def wrap_page(page: str) -> str:
+    return _head + mixin_tag(page, "main") + _footer + _tail

tools/page_shares.py

@@ -1,65 +0,0 @@
-
-HEAD_EMBED = """
-
-<div class="container">
-<nav class="custom-nav">
-<ul>
-<li><a href="/"><strong>mjestečko</strong></a></li>
-<li><a href="https://git.poto.cafe/veclavtalica/mjestecko">source</a></li>
-<li><a href="/feed.xml">rss</a></li>
-<li><a href="/articles/mjestečko.html">about</a></li>
-<li><a href="https://modarchive.org/index.php?request=view_artist_modules&query=96070">tracks</a></li>
-<li><a rel="me" href="https://poto.cafe/@veclavtalica">mastodon</a></li>
-</ul>
-</nav>
-
-"""
-
-NOTICE = """
-<footer>
-<a href="#top">Take me home ↑</a>
-</footer>
-
-"""
-
-TAIL_EMBED = """
-
-</div>
-
-"""
-
-ADJECTIVES = ["*wild*", "**wacky**", "very humble", "**most serious**"]
-
-MONTHS = {
-    1: "January",
-    2: "February",
-    3: "March",
-    4: "April",
-    5: "May",
-    6: "June",
-    7: "July",
-    8: "August",
-    9: "September",
-    10: "October",
-    11: "November",
-    12: "December"
-}
-
-WEEKDAYS = {
-    0: "Monday",
-    1: "Tuesday",
-    2: "Wednesday",
-    3: "Thursday",
-    4: "Friday",
-    5: "Saturday",
-    6: "Sunday"
-}
-
-def mixin_tag(content: str, tag: str) -> str:
-    return f"""<{tag}>
-{content}</{tag}>
-
-"""
-
-def wrap_page(page: str) -> str:
-    return HEAD_EMBED + mixin_tag(page, "main") + NOTICE + TAIL_EMBED

tools/plaintext_article_listing_generator.py

@@ -0,0 +1,15 @@
+#!/usr/bin/env python3
+
+from sys import argv, exit
+
+from article_utils import parse_article_directory, sort_titles_by_date
+
+if len(argv) <= 1:
+    print("No directory was supplied")
+    exit(-1)
+
+articles = parse_article_directory(argv[1])
+
+result = '\n'.join(sort_titles_by_date(articles))
+
+print(result)

tools/rfc822.py

@@ -1,4 +1,4 @@
-from page_shares import MONTHS, WEEKDAYS
+from date_descriptions import MONTHS, WEEKDAYS
 
 def stringify_date(date) -> str:
     return f"{WEEKDAYS[date.tm_wday][:3]}, {date.tm_mday} {MONTHS[date.tm_mon][:3]} {date.tm_year} {date.tm_hour:02d}:{date.tm_min:02d}:{date.tm_sec:02d} GMT"

tools/tag_listing_generator.py

@@ -0,0 +1,61 @@
+#!/usr/bin/env python3
+
+# todo: Problems might arise with casing if there are overlaps such as 'Tag' and 'tag'.
+
+from sys import argv, exit
+import time, urllib.parse, re
+from os import walk
+import os.path as path
+
+from article_utils import the_line_after_metadata, parse_metadata
+from page_builder import wrap_page
+
+# todo: Reuse
+tag_listing_header = """CSS:    /style.css
+HTML header:    <meta name="viewport" content="width=device-width, initial-scale=1">
+
+"""
+main_listing_header = """CSS:    /style.css
+HTML header:    <meta name="viewport" content="width=device-width, initial-scale=1">
+
+"""
+
+if len(argv) <= 1:
+    print("No article directory was supplied")
+    exit(-1)
+
+if len(argv) <= 2:
+    print("No tag listing output directory was supplied")
+    exit(-1)
+
+tag_to_tag_page = {}
+tag_to_articles = {}
+tag_counts = {}
+article_to_title = {}
+
+for root, dirs, _ in walk(argv[1]):
+    for d in dirs:
+        metadata = parse_metadata(path.abspath(root + '/' + d + "/page.mmd"))
+        article = "/articles/" + urllib.parse.quote(d) + ".html"
+        for tag in metadata.get('Tags', []):
+            tag_to_articles[tag] = tag_to_articles.get(tag, []) + [article]
+            tag_counts[tag] = tag_counts.get(tag, 0) + 1
+            article_to_title[article] = metadata['Title']
+    break
+
+for tag in tag_to_articles:
+    tag_page = f"/tags/{urllib.parse.quote(tag.lower())}.html"
+    tag_to_tag_page[tag] = tag_page
+
+    with open(argv[2] + tag_page, 'w') as f:
+        tagged_article_listing = f"\n# Tagged {tag} #\n---\n" + \
+            '\n'.join(f"- [{article_to_title[article]}]({article})" \
+                for article in tag_to_articles[tag])
+        f.write(tag_listing_header + wrap_page(tagged_article_listing))
+
+main_listing = "\n# Tag Listing #\n---\n" + \
+    ', '.join(f"[{tag}]({tag_to_tag_page[tag]}) ({tag_counts[tag]})" \
+        for tag in sorted(tag_counts.keys(), key=lambda x: tag_counts[x], reverse=True)) + \
+    '\n\n'
+
+print(main_listing_header + wrap_page(main_listing))

tools/widgets/list_selector.py

@@ -0,0 +1,95 @@
+#!/usr/bin/env python3
+
+from sys import argv
+import curses
+
+from wrapper import widget_wrapper
+
+list_starting_arg = 1
+for i, arg in enumerate(argv[1:]):
+    if arg == '--':
+        if i + 2 == len(argv):
+            print("Empty list given")
+            exit(-1)
+        list_starting_arg = i + 2
+        break
+else:
+    print("List starting -- wasn't given")
+    exit(-1)
+
+desc_arg = ''
+result_arg = 'index'
+
+# todo: Generalize and simplify over descriptor object.
+for arg in argv[1:]:
+    if arg.startswith('--'):
+        if arg == '--':
+            break
+        elif arg.startswith('--result='):
+            result_arg = arg[arg.find('=') + 1:]
+            if result_arg not in ['index', 'line']:
+                print("Invalid --result=")
+                exit(-1)
+        elif arg.startswith('--desc='):
+            desc_arg = arg[arg.find('=') + 1:]
+        else:
+            print("Unknown parameter ", arg)
+            exit(-1)
+    else:
+        print("Unknown parameter ", arg)
+        exit(-1)
+
+current = 0
+lines = argv[list_starting_arg:]
+
+list_box = None
+
+def init(screen):
+    global list_box
+
+    curses.start_color()
+    curses.curs_set(0)
+
+    y = 0
+    if desc_arg != '':
+        list_box = screen.subwin(y + 1, 0)
+        y += 1
+
+def draw_list_box():
+    y = 0
+
+    list_box.border()
+    y += 1
+
+    for i, line in enumerate(lines):
+        list_box.addstr(y, 1, line, curses.A_REVERSE if i == current else curses.A_NORMAL)
+        y += 1
+
+    list_box.refresh()
+
+def driver(screen):
+    global current
+
+    y = 0
+
+    if desc_arg != '':
+        screen.addstr(y, 0, desc_arg)
+        y += 1
+
+    draw_list_box()
+
+    key = screen.getch()
+    if key == curses.KEY_DOWN:
+        current = (current + 1) % len(lines)
+    elif key == curses.KEY_UP:
+        current = len(lines) - 1 if current == 0 else current - 1
+    elif key == curses.KEY_ENTER or key == 10 or key == 13:
+        if result_arg == 'index':
+            return str(current)
+        elif result_arg == 'line':
+            return lines[current]
+
+    screen.refresh()
+
+if __name__ == "__main__":
+    print(widget_wrapper(init, driver))

tools/widgets/wrapper.py

@@ -0,0 +1,47 @@
+import curses
+import signal
+import os, sys
+from sys import argv, exit
+
+def handler(signum, frame):
+    curses.endwin()
+    exit(1)
+
+init = None
+driver = None
+
+def curses_wrapper(screen):
+    curses.noecho()
+    curses.cbreak()
+    screen.keypad(True)
+
+    init(screen)
+
+    while True:
+        result = driver(screen)
+        if result != None:
+            return result
+
+def widget_wrapper(p_init, p_driver):
+    signal.signal(signal.SIGINT, handler)
+
+    global init, driver
+    init = p_init
+    driver = p_driver
+
+    with open('/dev/tty', 'rb') as inf, open('/dev/tty', 'wb') as outf:
+        saved_stdin = os.dup(0)
+        saved_stdout = os.dup(1)
+        saved_stderr = os.dup(2)
+
+        os.dup2(inf.fileno(), 0)
+        os.dup2(outf.fileno(), 1)
+        os.dup2(outf.fileno(), 2)
+
+        result = curses.wrapper(curses_wrapper)
+
+        os.dup2(saved_stdin, 0)
+        os.dup2(saved_stdout, 1)
+        os.dup2(saved_stderr, 2)
+
+        return result

upload.sh

@@ -1,4 +1,14 @@
-#!/usr/bin/bash
+#!/usr/bin/env bash
+
+set +e
+
+for arg in "$*"
+do
+  case "$arg" in
+    "--fresh") find ./html/ -type f -name '*.upload-checksum' -delete
+      ;;
+  esac
+done
 
 for cur in ./html/{*,*/*,*/*/*}; do
   if [ -f "$cur" ] && [[ ! "$cur" == *.upload-checksum ]]; then