From 682f54728e1c6ca91a297f6620bc4a1ea883f08d Mon Sep 17 00:00:00 2001 From: ArgentumCation Date: Thu, 5 Sep 2024 22:24:20 -0400 Subject: [PATCH] lots of stuff idk --- auxin/configuration.nix | 90 +++- auxin/hardware-configuration.nix | 73 ++- common.nix | 1 + etc/seaweedfs.nix | 896 +++++++++++++++++++++++++++++++ flake.lock | 66 +-- flake.nix | 32 +- galanin/configuration.nix | 2 +- node.nix | 10 + orexin/configuration.nix | 2 +- primary.nix | 59 ++ 10 files changed, 1123 insertions(+), 108 deletions(-) create mode 100644 etc/seaweedfs.nix create mode 100644 node.nix create mode 100644 primary.nix diff --git a/auxin/configuration.nix b/auxin/configuration.nix index 28add55..44e3234 100644 --- a/auxin/configuration.nix +++ b/auxin/configuration.nix @@ -1,18 +1,22 @@ -{ - config, - lib, - pkgs, - ... -}: let +{ config +, lib +, pkgs +, ... +}: +let ports = [ + 111 # RPC? + 2049 # NFS 22 # SSH 22000 # syncthing 2222 # Gitea SSH 2377 25565 # Minecraft 3001 # Gitea(->3000) + 4001 # Lockd + 4002 # Mountd 443 # HTTPS - 445 + 445 # SMB 7359 # jellyfin (dlna?) 7946 80 # HTTP @@ -24,11 +28,9 @@ 8842 # Crowdsec (-> 8842) 8888 # Kodi 9000 # Portainer (-> 9443) - 2049 # NFS - 4001 # Lockd - 4002 # Mountd ]; -in { +in +{ imports = [ # Include the results of the hardware scan. # ./hardware-configuration.nix @@ -38,11 +40,26 @@ in { kodi.enableAdvancedLauncher = true; pulseaudio = true; }; + # overlays = [ + # (final: prev: { + # seaweedfs = prev.seaweedfs.overrideAttrs + # (old: { + # doCheck = false; + # postInstall = /*old.postInstall ++*/ '' + # mkdir -p $out/sbin + # cp -r $out/bin/weed $out/sbin/weed + # cp -r $out/bin/weed $out/sbin/mount.weed + # + # ''; + # }); + # } + # ) + # ]; }; networking = { # hostName = "auxin"; # Define your hostname. - nameservers = ["192.168.1.207"]; + nameservers = [ "192.168.1.1" ]; firewall = { enable = true; allowedUDPPorts = ports; @@ -63,6 +80,8 @@ in { prefixLength = 24; } ]; + interfaces.br0.macAddress = "0a:d5:3b:e0:88:bd"; + interfaces.enp9s0.macAddress = "0a:d5:3b:e0:88:bd"; defaultGateway = { address = "192.168.1.1"; }; @@ -80,6 +99,9 @@ in { gamescope lazygit nil + dig + nmap + seaweedfs nodePackages.nodejs nodePackages.pnpm ]; @@ -106,6 +128,7 @@ in { }; }; programs = { + fuse.userAllowOther = true; steam = { gamescopeSession.enable = true; enable = true; @@ -140,13 +163,21 @@ in { }; services = { - nfs.server = { - enable = true; - mountdPort = 4002; - lockdPort = 4001; - exports = '' - /raid/swarm 192.168.1.0/24(rw) - ''; + nfs = { + idmapd.settings = { + Mapping = { + "Nobody-User" = lib.mkForce "dockremap"; + "Nobody-Group" = lib.mkForce "dockremap"; + }; + }; + server = { + enable = true; + mountdPort = 4002; + lockdPort = 4001; + exports = '' + /raid/swarm 192.168.1.0/24(rw,sync,all_squash,anonuid=100000,anongid=100000) + ''; + }; }; gvfs.enable = true; xrdp = { @@ -202,9 +233,9 @@ in { }; }; }; - btrbk = {}; # TODO: - caddy = {}; # TODO: - ceph = {}; # TODO: + btrbk = { }; # TODO: + caddy = { }; # TODO: + ceph = { }; # TODO: fail2ban = { enable = true; ignoreIP = [ @@ -226,6 +257,11 @@ in { path = "/"; browseable = "yes"; "guest ok" = "yes"; + "read only" = "no"; + "create mask" = "0644"; + "directory mask" = "0755"; + "force user" = "dockremap"; + "force group" = "dockremap"; }; }; }; @@ -249,14 +285,15 @@ in { }; # TODO: - powerManagement.enable = lib.mkForce false; + powerManagement.enable = lib.mkForce + false; systemd = { targets = { sleep.enable = false; suspend.enable = false; }; timers."qbittorrent-healthcheck" = { - wantedBy = ["timers.target"]; + wantedBy = [ "timers.target" ]; timerConfig = { OnCalendar = "*/5 * * * *"; Unit = "qbittorrent-healthcheck"; @@ -277,7 +314,7 @@ in { unitConfig = { RequiresMountsFor = "/raid"; }; - wantedBy = ["graphical.target"]; + wantedBy = [ "graphical.target" ]; }; sshdAlert = { enable = true; @@ -285,11 +322,12 @@ in { Requires = "sshd.service"; }; serviceConfig = { + User = "dockremap"; ExecStart = '' ${pkgs.curl}/bin/curl --request POST --url https://discord.com/api/webhooks/1235751608046846012/CU7tz271Z3Rbq9mPV0_rB5RBCRDhLKhGH14ebBm-TePpWFqKKJaCRYVMHYTJsIaSq2H- --header 'Content-Type: application/json' --data '{"username": "Auxin SSH status","avatar_url": "https://pbs.twimg.com/media/GMPtuovaQAAQ7Qr?format=png&name=large","content": "SSHD is Running!"}' ''; }; - wantedBy = ["multi-user.target"]; + wantedBy = [ "multi-user.target" ]; }; }; }; diff --git a/auxin/hardware-configuration.nix b/auxin/hardware-configuration.nix index f424953..636c25b 100644 --- a/auxin/hardware-configuration.nix +++ b/auxin/hardware-configuration.nix @@ -1,12 +1,11 @@ # Do not modify this file! It was generated by ‘nixos-generate-config’ # and may be overwritten by future invocations. Please make changes # to /etc/nixos/configuration.nix instead. -{ - config, - lib, - pkgs, - modulesPath, - ... +{ config +, lib +, pkgs +, modulesPath +, ... }: { imports = [ (modulesPath + "/installer/scan/not-detected.nix") @@ -18,32 +17,32 @@ systemd-boot.enable = true; efi.canTouchEfiVariables = true; }; - initrd.availableKernelModules = ["xhci_pci" "ahci" "nvme" "usb_storage" "usbhid" "sd_mod"]; - initrd.kernelModules = []; - kernelModules = ["kvm-intel"]; - extraModulePackages = []; + initrd.availableKernelModules = [ "xhci_pci" "ahci" "nvme" "usb_storage" "usbhid" "sd_mod" ]; + initrd.kernelModules = [ ]; + kernelModules = [ "kvm-intel" ]; + extraModulePackages = [ ]; }; fileSystems."/" = { device = "/dev/disk/by-uuid/b6931dc5-1ccb-4d81-abf6-38b6febc6176"; fsType = "btrfs"; - options = ["subvol=@nixos" "noatime"]; + options = [ "subvol=@nixos" "noatime" ]; }; fileSystems."/nix" = { device = "/dev/disk/by-uuid/b6931dc5-1ccb-4d81-abf6-38b6febc6176"; fsType = "btrfs"; - options = ["subvol=@nix" "noatime"]; + options = [ "subvol=@nix" "noatime" ]; }; fileSystems."/home" = { device = "/dev/disk/by-uuid/b6931dc5-1ccb-4d81-abf6-38b6febc6176"; fsType = "btrfs"; - options = ["subvol=@home" "noatime" "compress=zstd"]; + options = [ "subvol=@home" "noatime" "compress=zstd" ]; }; fileSystems."/swap" = { device = "/dev/disk/by-uuid/b6931dc5-1ccb-4d81-abf6-38b6febc6176"; fsType = "btrfs"; - options = ["subvol=@swap"]; + options = [ "subvol=@swap" ]; }; fileSystems."/boot" = { @@ -54,11 +53,52 @@ fileSystems."/raid" = { device = "/dev/sdb1"; fsType = "btrfs"; - options = ["noatime" "space_cache=v2" "compress=zstd"]; + options = [ "noatime" "space_cache=v2" "compress=zstd" ]; }; + # 96 │ localhost:4208:/ /weed fuse.seaweedfs rw,nosuid,nodev,relatime,user_id=1000,group_id=1000,default_permissions,allow_other,max_read=1048576 0 0 + # 31 │ /dev/sda1 /raid btrfs rw,noatime,compress=zstd:3,discard=async,space_cache=v2,subvolid=5,subvol=/ 0 0 + # fileSystems."/weed" = { + # device = "fuse"; + # fsType = "weed"; + # options = [ "filer=localhost:4208" "filer.path=/" "nofail" "_netdev" "x-systemd.mount-timeout=5" "x-systemd.automount" ]; + # }; + systemd.services."seaweedfs-mount" = { + wantedBy = [ "multi-user.target" ]; + after = [ "network.target" ]; + requires = [ "seaweedfs-default-filer-main.service" ]; + serviceConfig = { + ExecStartPre = [ + "${pkgs.coreutils}/bin/mkdir -m 0775 -pv /weed" + "${pkgs.e2fsprogs}/bin/chattr +i /weed" # Stop files being accidentally written to unmounted directory + ]; + ExecStart = "${pkgs.seaweedfs}/bin/weed mount -filer=localhost:4208 -dir=/weed -filer.path=/"; + ExecStopPost = "-${pkgs.fuse}/bin/fusermount -u /weed"; + KillMode = "process"; + Restart = "on-failure"; + }; + }; # swapDevices = [ "/swap/swapfile" ]; - + # systemd.mounts = [{ + # # what = "fuse"; + # where = "/weed"; + # type = "fuse.seaweedfs"; + # options = "defaults"; + # unitConfig = { + # Requires = [ "seaweedfs-default-filer-main.service" ]; + # After = [ "network.target" ]; + # ExecMount = "${pkgs.seaweedfs}/bin/weed mount -dir=/weed -filer=localhost:4208"; + # + # }; + # + # }]; + # systemd.automounts = [{ + # wantedBy = [ "multi-user.target" ]; + # automountConfig = { + # TimeoutIdleSec = "5"; + # }; + # where = "/weed"; + # }]; # Enables DHCP on each ethernet and wireless interface. In case of scripted networking # (the default) this is the recommended approach. When using systemd-networkd it's # still possible to use this option, but it's recommended to use it in conjunction @@ -75,3 +115,4 @@ pulseaudio.support32Bit = true; ## If compatibility with 32-bit applications is desired. }; } + diff --git a/common.nix b/common.nix index 0a4c2c0..42f7985 100644 --- a/common.nix +++ b/common.nix @@ -151,6 +151,7 @@ in { brightnessctl zoxide zsh + lsof ]; programs = { tmux.enable = true; diff --git a/etc/seaweedfs.nix b/etc/seaweedfs.nix new file mode 100644 index 0000000..133665a --- /dev/null +++ b/etc/seaweedfs.nix @@ -0,0 +1,896 @@ +{ config, lib, pkgs, ... }: + +with lib; + +let + cfg = config.services.seaweedfs; + + clusterModule = cluster: { + options = { + package = mkOption { + type = types.package; + default = pkgs.seaweedfs; + }; + + security.grpc = + let + auth = mkOption { + type = with types; nullOr (submodule { + options = { + cert = mkOption { type = path; }; + key = mkOption { type = path; }; + }; + }); + default = null; + }; + in + { + ca = mkOption { + type = with types; nullOr str; + default = null; + }; + + master = auth; + volume = auth; + filer = auth; + client = auth; + msgBroker = auth; + }; + + masters = mkOption { + type = with types; attrsOf (submodule (masterModule cluster.config)); + default = { }; + description = "SeaweedFS masters"; + }; + + volumes = mkOption { + type = with types; attrsOf (submodule (volumeModule cluster.config)); + default = { }; + description = "SeaweedFS volumes"; + }; + + filers = mkOption { + type = with types; attrsOf (submodule (filerModule cluster.config)); + default = { }; + description = "SeaweedFS filers"; + }; + + webdavs = mkOption { + type = with types; attrsOf (submodule (webdavModule cluster.config)); + default = { }; + description = "SeaweedFS WebDAV servers"; + }; + + instances = mkOption { + type = with types; attrsOf (submodule instanceModule); + description = "SeaweedFS instances"; + default = + mapAttrs' + (name: master: nameValuePair + "master-${name}" + { + inherit (master) cluster configs; + + command = "master"; + + args = with master; + [ + "-port=${toString port}" + "-volumeSizeLimitMB=${toString volumeSizeLimitMB}" + ] ++ + optional (cpuprofile != "") "-cpuprofile=${cpuprofile}" ++ + optional (defaultReplication != null) ("-defaultReplication=${defaultReplication.code}") ++ + optional disableHttp "-disableHttp" ++ + optional (garbageThreshold != "") "-garbageThreshold=${garbageThreshold}" ++ + optional (ip != "") "-ip=${ip}" ++ + optional (master."ip.bind" != "") "-ip.bind=${master."ip.bind"}" ++ + optional (mdir != "") "-mdir=${mdir}" ++ + optional (memprofile != "") "-memprofile=${memprofile}" ++ + optional metrics.enable "-metrics.address=${metrics.address.text}" ++ + optional (metrics.intervalSeconds != null) "-metrics.intervalSeconds=${toString metrics.intervalSeconds}" ++ + optional (peers != [ ]) ("-peers=" + (concatStringsSep "," (map (peer: peer.text) peers))) ++ + optional resumeState "-resumeState" ++ + optional volumePreallocate "-volumePreallocate" ++ + optional (whiteList != [ ]) ("-whiteList=" + (concatStringsSep "," whiteList)); + } + ) + cluster.config.masters // + mapAttrs' + (name: volume: nameValuePair + "volume-${name}" + { + inherit (volume) cluster configs; + + command = "volume"; + + args = with volume; + [ + "-port=${toString port}" + "-dir=${concatStringsSep "," dir}" + "-disk=${disk}" + "-fileSizeLimitMB=${toString fileSizeLimitMB}" + "-idleTimeout=${toString idleTimeout}" + "-index=${index}" + "-minFreeSpacePercent=${toString minFreeSpacePercent}" + "-preStopSeconds=${toString preStopSeconds}" + ] ++ + optional (compactionMBps != null) ("-compactionMBps=${compactionMBps}") ++ + optional (cpuprofile != "") "-cpuprofile=${cpuprofile}" ++ + optional (dataCenter != "") "-dataCenter=${dataCenter}" ++ + optional volume."images.fix.orientation" "-images.fix.orientation" ++ + optional (ip != "") "-ip=${ip}" ++ + optional (volume."ip.bind" != "") "-ip.bind=${volume."ip.bind"}" ++ + optional (max != [ ]) "-max=${concatStringsSep "," (map toString max)}" ++ + optional (memprofile != "") "-memprofile=${memprofile}" ++ + optional (metricsPort != null) "-metricsPort=${toString metricsPort}" ++ + optional (mserver != [ ]) ("-mserver=" + (concatStringsSep "," (map (mserver: mserver.text) mserver))) ++ + optional (volume."port.public" != null) "-port.public=${toString volume."port.public"}" ++ + optional pprof "-pprof" ++ + optional (publicUrl != "") "-publicUrl=${publicUrl}" ++ + optional (rack != "") "-rack=${rack}" ++ + optional (!volume."read.redirect") "-read.redirect=false" ++ + optional (whiteList != [ ]) ("-whiteList=" + (concatStringsSep "," whiteList)); + + systemdService.preStart = "mkdir -p ${concatStringsSep " " volume.dir}"; + } + ) + cluster.config.volumes // + mapAttrs' + (name: filer: nameValuePair + "filer-${name}" + { + inherit (filer) cluster configs; + + command = "filer"; + + args = with filer; + [ + "-port=${toString port}" + "-dirListLimit=${toString dirListLimit}" + "-maxMB=${toString maxMB}" + ] ++ + optional (collection != "") "-collection=${collection}" ++ + optional (dataCenter != "") "-dataCenter=${dataCenter}" ++ + optional (defaultReplicaPlacement != null) ("-defaultReplicaPlacement=${defaultReplicaPlacement.code}") ++ + optional disableDirListing "-disableDirListing" ++ + optional disableHttp "-disableHttp" ++ + optional encryptVolumeData "-encryptVolumeData" ++ + optional (ip != "") "-ip=${ip}" ++ + optional (filer."ip.bind" != "") "-ip.bind=${filer."ip.bind"}" ++ + optional (master != [ ]) ("-master=" + (concatStringsSep "," (map (master: master.text) master))) ++ + optional (metricsPort != null) "-metricsPort=${toString metricsPort}" ++ + optional (peers != [ ]) ("-peers=" + (concatStringsSep "," (map (peer: peer.text) peers))) ++ + optional (filer."port.readonly" != null) "-port.readonly=${toString filer."port.readonly"}" ++ + optional (rack != "") "-rack=${rack}" ++ + optionals s3.enable [ + "-s3" + "-s3.port=${toString filer.s3.port}" + ] ++ + optional (s3.enable && s3."cert.file" != "") "-s3.cert.file=${s3."cert.file"}" ++ + optional (s3.enable && s3."key.file" != "") "-s3.key.file=${s3."key.file"}" ++ + optional (s3.enable && s3.config != "") "-s3.config=${s3.config}" ++ + optional (s3.enable && s3.domainName != [ ]) "-s3.domainName=${concatStringsSep "," s3.domainName}"; + + systemdService.preStart = + let + conf = filer.configs.filer.leveldb2 or { }; + in + optionalString (conf ? "dir") "mkdir -p ${conf.dir}"; + } + ) + cluster.config.filers // + mapAttrs' + (name: webdav: nameValuePair + "webdav-${name}" + { + inherit (webdav) cluster; + + command = "webdav"; + + args = with webdav; + [ + "-port=${toString port}" + "-filer=${filer.text}" + "-cacheCapacityMB=${toString cacheCapacityMB}" + ] ++ + optional (collection != "") "-collection=${collection}" ++ + optional (cacheDir != "") "-cacheDir=${cacheDir}"; + } + ) + cluster.config.webdavs; + }; + }; + }; + + commonModule = cluster: common: { + options = { + cluster = mkOption { + type = types.submodule clusterModule; + internal = true; + }; + + openFirewall = mkEnableOption "open the firewall"; + }; + + config = { inherit cluster; }; + }; + + masterModule = cluster: master: { + imports = [ (commonModule cluster) ]; + + options = { + configs = mkOption { + type = with types; attrsOf attrs; + default.master.maintenance = { + scripts = '' + ec.encode -fullPercent=95 -quietFor=1h + ec.avenge -force + ec.balance -force + volume.balance -force + volume.fix.replication + ''; + sleep_minutes = 17; + }; + }; + + cpuprofile = mkOption { + type = types.str; + default = ""; + }; + + defaultReplication = mkOption { + type = types.submodule replicationModule; + default = { }; + }; + + disableHttp = mkEnableOption "disable HTTP requests, gRPC only"; + + garbageThreshold = mkOption { + type = types.str; + default = ""; + }; + + ip = mkOption { + type = types.str; + default = config.networking.hostName; + }; + + "ip.bind" = mkOption { + type = types.str; + default = "0.0.0.0"; + }; + + mdir = mkOption { + type = types.str; + default = "."; + }; + + memprofile = mkOption { + type = types.str; + default = ""; + }; + + metrics = { + enable = mkEnableOption "Prometheus"; + + address = mkOption { + type = types.submodule ipPortModule; + default = { }; + }; + + intervalSeconds = mkOption { + type = types.ints.unsigned; + default = 15; + }; + }; + + peers = mkOption { + type = peersType; + default = mapAttrsIpPort master.config.cluster.masters; + }; + + port = mkOption { + type = types.port; + default = 9333; + }; + + resumeState = mkEnableOption "resume previous state on master server"; + + volumePreallocate = mkEnableOption "preallocate disk space for volumes"; + + volumeSizeLimitMB = mkOption { + type = types.ints.unsigned; + default = 30000; + }; + + whiteList = mkOption { + type = with types; listOf str; + default = [ ]; + }; + }; + }; + + volumeModule = cluster: volume: { + imports = [ (commonModule cluster) ]; + + options = { + configs = mkOption { + type = with types; attrsOf attrs; + default = { }; + }; + + compactionMBps = mkOption { + type = with types; nullOr ints.unsigned; + default = null; + }; + + cpuprofile = mkOption { + type = types.str; + default = ""; + }; + + dataCenter = mkOption { + type = types.str; + default = ""; + }; + + dir = mkOption { + type = with types; listOf str; + default = [ "/var/lib/seaweedfs/${cluster._module.args.name}/volume-${volume.config._module.args.name}" ]; + }; + + disk = mkOption { + type = types.str; + default = "hdd"; + }; + fileSizeLimitMB = mkOption { + type = types.ints.unsigned; + default = 256; + }; + + idleTimeout = mkOption { + type = types.ints.unsigned; + default = 30; + }; + + "images.fix.orientation" = mkEnableOption "adjustment of jpg orientation when uploading"; + + index = mkOption { + type = types.enum [ + "memory" + "leveldb" + "leveldbMedium" + "leveldbLarge" + ]; + default = "memory"; + }; + + ip = mkOption { + type = types.str; + default = config.networking.hostName; + }; + + "ip.bind" = mkOption { + type = types.str; + default = "0.0.0.0"; + }; + + max = mkOption { + type = with types; listOf ints.unsigned; + default = [ 8 ]; + }; + + memprofile = mkOption { + type = types.str; + default = ""; + }; + + metricsPort = mkOption { + type = with types; nullOr port; + default = null; + }; + + minFreeSpacePercent = mkOption { + type = types.ints.unsigned; + default = 1; + }; + + mserver = mkOption { + type = peersType; + default = mapAttrsIpPort volume.config.cluster.masters; + }; + + port = mkOption { + type = types.port; + default = 8080; + }; + + "port.public" = mkOption { + type = with types; nullOr port; + default = null; + }; + + pprof = mkEnableOption "pprof http handlers. precludes -memprofile and -cpuprofile"; + + preStopSeconds = mkOption { + type = types.int; + default = 10; + }; + + publicUrl = mkOption { + type = types.str; + default = ""; + }; + + rack = mkOption { + type = types.str; + default = ""; + }; + + "read.redirect" = mkOption { + type = types.bool; + default = true; + }; + + whiteList = mkOption { + type = with types; listOf str; + default = [ ]; + }; + }; + }; + + filerModule = cluster: filer: { + imports = [ (commonModule cluster) ]; + + options = { + configs = mkOption { + type = with types; attrsOf attrs; + default.filer.leveldb2 = { + enabled = true; + dir = "/var/lib/seaweedfs/${cluster._module.args.name}/filer-${filer.config._module.args.name}/filerldb2"; + }; + }; + + collection = mkOption { + type = types.str; + default = ""; + }; + + dataCenter = mkOption { + type = types.str; + default = ""; + }; + + defaultReplicaPlacement = mkOption { + type = with types; nullOr (submodule replicationModule); + default = null; + }; + + dirListLimit = mkOption { + type = types.ints.unsigned; + default = 100000; + }; + + disableDirListing = mkEnableOption "turn off directory listing"; + + disableHttp = mkEnableOption "disable http request, only gRpc operations are allowed"; + + encryptVolumeData = mkEnableOption "encrypt data on volume servers"; + + ip = mkOption { + type = types.str; + default = config.networking.hostName; + }; + + "ip.bind" = mkOption { + type = types.str; + default = "0.0.0.0"; + }; + + master = mkOption { + type = peersType; + default = mapAttrsIpPort filer.config.cluster.masters; + }; + + maxMB = mkOption { + type = types.ints.unsigned; + default = 32; + }; + + metricsPort = mkOption { + type = with types; nullOr port; + default = null; + }; + + peers = mkOption { + type = peersType; + default = mapAttrsIpPort filer.config.cluster.filers; + }; + + port = mkOption { + type = types.port; + default = 8888; + }; + + "port.readonly" = mkOption { + type = with types; nullOr port; + default = null; + }; + + rack = mkOption { + type = types.str; + default = ""; + }; + + s3 = { + enable = mkEnableOption "whether to start S3 gateway"; + + "cert.file" = mkOption { + type = types.path; + default = ""; + }; + + config = mkOption { + type = types.path; + default = ""; + }; + + domainName = mkOption { + type = with types; listOf str; + default = [ ]; + }; + + "key.file" = mkOption { + type = types.path; + default = ""; + }; + + port = mkOption { + type = types.port; + default = 8333; + }; + }; + }; + }; + + webdavModule = cluster: webdav: { + imports = [ (commonModule cluster) ]; + + options = { + cacheCapacityMB = mkOption { + type = types.int; + default = 1000; + }; + + cacheDir = mkOption { + type = types.str; + default = "."; + }; + + collection = mkOption { + type = types.str; + default = ""; + }; + + filer = mkOption { + type = types.submodule ipPortModule; + default = { + ip = "127.0.0.1"; + port = 8888; + }; + }; + + port = mkOption { + type = types.port; + default = 7333; + }; + }; + }; + + instanceModule = instance: { + options = { + cluster = mkOption { + type = types.submodule clusterModule; + internal = true; + }; + + command = mkOption { + type = types.enum [ + "server" + "master" + "volume" + "mount" + "filer" + "filer.replicate" + "filer.sync" + "s3" + "msgBroker" + "watch" + "webdav" + ]; + }; + + logArgs = mkOption { + type = with types; listOf str; + default = [ ]; + }; + + args = mkOption { + type = with types; listOf str; + default = [ ]; + }; + + configs = mkOption { + type = with types; attrsOf attrs; + default = { }; + }; + + package = mkOption { + type = types.package; + default = instance.config.cluster.package; + }; + + systemdService = mkOption { + type = types.attrs; + default = { }; + }; + }; + + config = { + logArgs = [ "-logtostderr" ]; + + systemdService.path = optional (instance.config.command == "mount") pkgs.fuse; + }; + }; + + replicationModule = replication: { + options = { + dataCenter = mkOption { + type = types.ints.between 0 9; + default = 0; + }; + + rack = mkOption { + type = types.ints.between 0 9; + default = 0; + }; + + server = mkOption { + type = types.ints.between 0 9; + default = 0; + }; + + code = mkOption { + readOnly = true; + internal = true; + type = types.str; + default = with replication.config; "${toString dataCenter}${toString rack}${toString server}"; + }; + }; + }; + + peersType = with types; listOf (submodule ipPortModule); + + ipPortModule = ipPort: { + options = { + ip = mkOption { + type = types.str; + }; + + port = mkOption { + type = types.port; + }; + + text = mkOption { + internal = true; + readOnly = true; + type = types.str; + default = with ipPort.config; "${ip}:${toString port}"; + }; + }; + }; + + mapAttrsIpPort = attrs: mapAttrsToList (name: value: { inherit (value) ip port; }) attrs; + + toTOML = with generators; toINI { + mkKeyValue = mkKeyValueDefault + { + mkValueString = v: + if isString v + then + ( + if hasInfix "\n" v + then '' + """ + ${removeSuffix "\n" v} + """ + '' + else ''"${v}"'' + ) + else mkValueStringDefault { } v; + } "="; + }; + + flattenAttrs = separator: attrs: + let + /* + attrs = { + a = { + m1 = {}; + m2 = {}; + }; + b = { + m1 = {}; + }; + } + */ + + /* + step1 = { + a = [ + { name = "a-m1"; value = {}; } + { name = "a-m2"; value = {}; } + ]; + b = [ + { name = "b-m1"; value = {}; } + ]; + }; + */ + step1 = mapAttrs + (outerName: outerValues: + mapAttrsToList + (innerName: innerValues: nameValuePair + "${outerName}${separator}${innerName}" + innerValues + ) + outerValues + ) + attrs; + + /* + step2 = [ + [ + { name = "a-m1"; value = {}; } + { name = "a-m2"; value = {}; } + ] + [ + { name = "b-m1"; value = {}; } + ] + ]; + */ + step2 = mapAttrsToList (name: value: value) step1; + + /* + step3 = [ + { name = "a-m1"; value = {}; } + { name = "a-m2"; value = {}; } + { name = "b-m1"; value = {}; } + ]; + */ + step3 = flatten step2; + in + /* + { + a-m1 = {}; + a-m2 = {}; + b-m1 = {}; + }; + */ + builtins.listToAttrs step3; +in +{ + options.services.seaweedfs = { + clusters = mkOption { + type = with types; attrsOf (submodule clusterModule); + default = { }; + description = "SeaweedFS clusters"; + }; + }; + + config = { + systemd.services = mapAttrs' + (name: instance: nameValuePair "seaweedfs-${name}" instance) + (flattenAttrs "-" ( + mapAttrs + (clusterName: cluster: + mapAttrs + (instanceName: instance: with instance; recursiveUpdate systemdService rec { + description = "SeaweedFS ${clusterName} ${instanceName}"; + wants = [ "network.target" ]; + after = wants; + wantedBy = [ "multi-user.target" ]; + preStart = with serviceConfig; '' + ${ + let securityFile = config.environment.etc."seaweedfs/${clusterName}/security.toml"; + in optionalString securityFile.enable "ln -s /etc/${securityFile.target} ${WorkingDirectory}/" + } + + # TODO replace find usage with statically known condition + find -L /etc/${ConfigurationDirectory} -type f -exec ln -s '{}' ${WorkingDirectory}/ \; + + ${optionalString (systemdService ? preStart) systemdService.preStart} + ''; + serviceConfig = rec { + User = "dockremap"; #todo: move out of this file + ExecStart = "${package}/bin/weed ${concatStringsSep " " logArgs} ${command} ${concatStringsSep " " args}"; + Restart = "on-failure"; + Type = "exec"; + ConfigurationDirectory = "seaweedfs/${clusterName}/${instanceName}"; + RuntimeDirectory = ConfigurationDirectory; + RuntimeDirectoryPreserve = "restart"; + WorkingDirectory = "/run/${RuntimeDirectory}"; + }; + }) + cluster.instances + ) + cfg.clusters + )); + + environment.etc = + (mapAttrs' + (name: cluster: + let file = "seaweedfs/${name}/security.toml"; + in + nameValuePair file { + enable = config.environment.etc.${file}.text != ""; + text = with cluster.security.grpc; toTOML ( + (if ca == null then { } else { grpc.ca = ca; }) // + (if master == null then { } else { "grpc.master" = { inherit (master) cert key; }; }) // + (if volume == null then { } else { "grpc.volume" = { inherit (volume) cert key; }; }) // + (if filer == null then { } else { "grpc.filer" = { inherit (filer) cert key; }; }) // + (if client == null then { } else { "grpc.client" = { inherit (client) cert key; }; }) // + (if msgBroker == null then { } else { "grpc.msg_broker" = { inherit (msgBroker) cert key; }; }) + ); + } + ) + cfg.clusters) // + (mapAttrs' + (name: config: nameValuePair + "seaweedfs/${name}.toml" + { text = toTOML config; } + ) + (flattenAttrs "/" ( + mapAttrs + (clusterName: cluster: + flattenAttrs "/" ( + mapAttrs + (instanceName: instance: instance.configs) + cluster.instances + ) + ) + cfg.clusters + )) + ); + + networking.firewall.allowedTCPPorts = + let + modulesToPorts = extraPorts: mapAttrsToList (name: module: + with module; + optionals openFirewall ( + [ port (port + 10000) ] ++ + (filter (p: p != null) (extraPorts module)) + ) + ); + in + flatten (mapAttrsToList + (clusterName: cluster: + modulesToPorts + (master: [ ]) + cluster.masters ++ + + modulesToPorts + (volume: with volume; [ metricsPort volume."port.public" ]) + cluster.volumes ++ + + modulesToPorts + (filer: with filer; [ metricsPort filer."port.readonly" s3.port ]) + cluster.filers ++ + + modulesToPorts + (webdav: [ ]) + cluster.webdavs + ) + cfg.clusters); + }; +} + diff --git a/flake.lock b/flake.lock index a8aafc4..d8999be 100644 --- a/flake.lock +++ b/flake.lock @@ -40,11 +40,11 @@ ] }, "locked": { - "lastModified": 1719994518, - "narHash": "sha256-pQMhCCHyQGRzdfAkdJ4cIWiw+JNuWsTX7f0ZYSyz0VY=", + "lastModified": 1722555600, + "narHash": "sha256-XOQkdLafnb/p9ij77byFQjDf5m5QYl9b2REiVClC+x4=", "owner": "hercules-ci", "repo": "flake-parts", - "rev": "9227223f6d922fee3c7b190b2cc238a99527bbb7", + "rev": "8471fe90ad337a8074e957b69ca4d0089218391d", "type": "github" }, "original": { @@ -88,11 +88,11 @@ ] }, "locked": { - "lastModified": 1721042469, - "narHash": "sha256-6FPUl7HVtvRHCCBQne7Ylp4p+dpP3P/OYuzjztZ4s70=", + "lastModified": 1724857454, + "narHash": "sha256-Qyl9Q4QMTLZnnBb/8OuQ9LSkzWjBU1T5l5zIzTxkkhk=", "owner": "cachix", "repo": "git-hooks.nix", - "rev": "f451c19376071a90d8c58ab1a953c6e9840527fd", + "rev": "4509ca64f1084e73bc7a721b20c669a8d4c5ebe6", "type": "github" }, "original": { @@ -132,11 +132,11 @@ ] }, "locked": { - "lastModified": 1719226092, - "narHash": "sha256-YNkUMcCUCpnULp40g+svYsaH1RbSEj6s4WdZY/SHe38=", + "lastModified": 1724947644, + "narHash": "sha256-MHHrHasTngp7EYQOObHJ1a/IsRF+wodHqOckhH6uZbk=", "owner": "hercules-ci", "repo": "hercules-ci-effects", - "rev": "11e4b8dc112e2f485d7c97e1cee77f9958f498f5", + "rev": "dba4367b9a9d9615456c430a6d6af716f6e84cef", "type": "github" }, "original": { @@ -155,11 +155,11 @@ "nixpkgs": "nixpkgs" }, "locked": { - "lastModified": 1722471252, - "narHash": "sha256-rgNPBiWN+y2jBiXIEgV/McBfkPeZax80eol0FBoupCk=", + "lastModified": 1724996444, + "narHash": "sha256-bgDfNsVPleUyx6vNr5INJTLfkLycNmL3yvSBv1OguLs=", "owner": "nix-community", "repo": "neovim-nightly-overlay", - "rev": "fe7178b41d84add25c63a695620629050fb35bbd", + "rev": "d0f68c980e3a0a3a8e63ccca93a01f87fb77937e", "type": "github" }, "original": { @@ -171,11 +171,11 @@ "neovim-src": { "flake": false, "locked": { - "lastModified": 1722463651, - "narHash": "sha256-3YorBqxT1RpL3Z2rLDCJhG+1HnBsgjrW8AOlTkFWlbA=", + "lastModified": 1724970905, + "narHash": "sha256-6HqoxweeX3tQbchJpjUNiBKj/2P3oiQBR42B/QuB+a0=", "owner": "neovim", "repo": "neovim", - "rev": "e820474cde09273608be5f57e1032aab21e3c97d", + "rev": "4353996d0fa8e5872a334d68196d8088391960cf", "type": "github" }, "original": { @@ -189,11 +189,11 @@ "nixpkgs": "nixpkgs_2" }, "locked": { - "lastModified": 1722445220, - "narHash": "sha256-PW5FRqLhqg0xGpPjY2Poa464tyBQiyKd0tQGZ0HnMiU=", + "lastModified": 1724994893, + "narHash": "sha256-yutISDGg6HUaZqCaa54EcsfTwew3vhNtt/FNXBBo44g=", "owner": "LnL7", "repo": "nix-darwin", - "rev": "7e08a9dd34314fb8051c28b231a68726c54daa7b", + "rev": "c8d3157d1f768e382de5526bb38e74d2245cad04", "type": "github" }, "original": { @@ -204,11 +204,11 @@ }, "nixos-unstable": { "locked": { - "lastModified": 1722185531, - "narHash": "sha256-veKR07psFoJjINLC8RK4DiLniGGMgF3QMlS4tb74S6k=", + "lastModified": 1724819573, + "narHash": "sha256-GnR7/ibgIH1vhoy8cYdmXE6iyZqKqFxQSVkFgosBh6w=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "52ec9ac3b12395ad677e8b62106f0b98c1f8569d", + "rev": "71e91c409d1e654808b2621f28a327acfdad8dc2", "type": "github" }, "original": { @@ -220,11 +220,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1722415718, - "narHash": "sha256-5US0/pgxbMksF92k1+eOa8arJTJiPvsdZj9Dl+vJkM4=", + "lastModified": 1724840184, + "narHash": "sha256-RXftd9nVNpCKHEaiMhAWiZo3U/SEdRPF0zD7s7u50Oc=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "c3392ad349a5227f4a3464dce87bcc5046692fce", + "rev": "4f9cb71da3ec4f76fd406a0d87a1db491eda6870", "type": "github" }, "original": { @@ -236,11 +236,11 @@ }, "nixpkgs-darwin": { "locked": { - "lastModified": 1722272837, - "narHash": "sha256-iHO942tXSkiZ0ZhWkfqCvqo9/67+S6WYfphXSJogEmM=", + "lastModified": 1725031139, + "narHash": "sha256-ejr32Vf+T7tz3Xo2LtQiQfEsmSEke5npKlhJuhSoY8U=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "89526a7d969e38fe8c30253170d44d0f131882de", + "rev": "33ce3a8d5f8b771344694667912c77233334f487", "type": "github" }, "original": { @@ -252,11 +252,11 @@ }, "nixpkgs-unstable": { "locked": { - "lastModified": 1722415718, - "narHash": "sha256-5US0/pgxbMksF92k1+eOa8arJTJiPvsdZj9Dl+vJkM4=", + "lastModified": 1725036679, + "narHash": "sha256-Ri79ZOEcZJFLr6+LgS3A0WYyroL/PqEuO+lI7u+G2tE=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "c3392ad349a5227f4a3464dce87bcc5046692fce", + "rev": "dac9db29e0e7ff2071ccc47b720aaffc3e74b504", "type": "github" }, "original": { @@ -282,11 +282,11 @@ }, "nixpkgs_3": { "locked": { - "lastModified": 1722221733, - "narHash": "sha256-sga9SrrPb+pQJxG1ttJfMPheZvDOxApFfwXCFO0H9xw=", + "lastModified": 1724855419, + "narHash": "sha256-WXHSyOF4nBX0cvHN3DfmEMcLOVdKH6tnMk9FQ8wTNRc=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "12bf09802d77264e441f48e25459c10c93eada2e", + "rev": "ae2fc9e0e42caaf3f068c1bfdc11c71734125e06", "type": "github" }, "original": { diff --git a/flake.nix b/flake.nix index a945873..9689b42 100644 --- a/flake.nix +++ b/flake.nix @@ -31,37 +31,6 @@ overlays = [inputs.neovim-nightly-overlay.overlays.default]; in { nixosConfigurations = { - auxin-iso = nixos-unstable.lib.nixosSystem { - modules = [ - "${nixos-unstable}/nixos/modules/installer/cd-dvd/installation-cd-minimal.nix" - "${nixos-unstable}/nixos/modules/installer/cd-dvd/channel.nix" - ./auxin/configuration.nix - ./vm.nix - ./common.nix - ]; - specialArgs = { - # inherit inputs; - meta = { - hostname = "auxin-iso"; - }; - }; - }; - auxin-vm = nixos-unstable.lib.nixosSystem { - modules = [ - # "${nixos-unstable}/nixos/modules/installer/cd-dvd/installation-cd-minimal.nix" - # "${nixos-unstable}/nixos/modules/installer/cd-dvd/channel.nix" - "${nixos-unstable}/nixos/modules/virtualisation/virtualbox-image.nix" - ./auxin/configuration.nix - ./vm.nix - ./common.nix - ]; - specialArgs = { - # inherit inputs; - meta = { - hostname = "auxin-vm"; - }; - }; - }; auxin = nixos-unstable.lib.nixosSystem { system = "x86_64-linux"; specialArgs = { @@ -71,6 +40,7 @@ ./auxin/configuration.nix ./auxin/hardware-configuration.nix ./common.nix + ./primary.nix ]; }; orexin = nixos-unstable.lib.nixosSystem { diff --git a/galanin/configuration.nix b/galanin/configuration.nix index cd94483..3257469 100644 --- a/galanin/configuration.nix +++ b/galanin/configuration.nix @@ -17,7 +17,7 @@ boot.loader.efi.canTouchEfiVariables = true; networking = { - nameservers = ["192.168.1.207"]; + # nameservers = ["192.168.1.207"]; firewall = { enable = false; }; diff --git a/node.nix b/node.nix new file mode 100644 index 0000000..bbcd767 --- /dev/null +++ b/node.nix @@ -0,0 +1,10 @@ +{ config +, lib +, pkgs +, meta +, ... +}: +let + foo = "bar"; +in +{ } diff --git a/orexin/configuration.nix b/orexin/configuration.nix index 5570473..433622f 100644 --- a/orexin/configuration.nix +++ b/orexin/configuration.nix @@ -17,7 +17,7 @@ boot.loader.efi.canTouchEfiVariables = true; networking = { - nameservers = ["192.168.1.207"]; + # nameservers = ["192.168.1.207"]; firewall = { enable = false; }; diff --git a/primary.nix b/primary.nix new file mode 100644 index 0000000..be18ebb --- /dev/null +++ b/primary.nix @@ -0,0 +1,59 @@ +inputs @ { config +, pkgs +, flakes +, ... +}: { + imports = [ + ./etc/seaweedfs.nix # the file from dermetfan + ]; + + services.seaweedfs.clusters.default = { + package = pkgs.seaweedfs; + + masters.main = { + openFirewall = true; + ip = "${config.networking.hostName}.lan"; + mdir = "/raid/@weed/metadata/"; + volumePreallocate = true; + + defaultReplication = { + dataCenter = 0; + rack = 0; + server = 0; + }; + }; + volumes.${config.networking.hostName} = { + openFirewall = true; + dataCenter = "ribosome"; + rack = "${config.networking.hostName}"; + ip = "${config.networking.hostName}.lan"; + dir = [ "/raid/@weed/volumes/volume_${config.networking.hostName}/" ]; + disk = "hdd"; # Replication gets screwy if these don't match + max = [ 0 ]; # use all space + port = 9334; + + mserver = [ + { + ip = "${config.networking.hostName}.lan"; + port = 9333; + } + ]; + }; + + filers.main = { + openFirewall = true; + dataCenter = "ribosome"; + encryptVolumeData = false; + ip = "${config.networking.hostName}.lan"; + peers = [ ]; + port = 4208; + master = [ + # this is actually in cluster.masters that I import in the real file + { + ip = "${config.networking.hostName}.lan"; + port = 9333; + } + ]; + }; + }; +}