diff --git a/src/swark/cedric.cljc b/src/swark/cedric.cljc deleted file mode 100644 index 8f51405..0000000 --- a/src/swark/cedric.cljc +++ /dev/null @@ -1,303 +0,0 @@ -(ns swark.cedric - {:added "0.1.4" - :doc "Protocol for persisting data as data driven EAV rows."} - (:require [clojure.edn :as edn] - [clojure.set :as set] - [clojure.data :as data] - #?(:cljs [goog.date :as gd]) - #?(:clj [clojure.java.io :as io]) - [clojure.data.csv :as csv] - [swark.atomic :as atomic] - [swark.core :as swark]) - #?(:clj (:import [java.time Instant]))) - -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;; CEDRIC - the Cedric Event DRIven datapersistence Companion -;; Store associatve data (maps) as rows in an append-only EAV database. - -;; TODO: Add headers to csv db file -;; TODO: Test in cljs as well -;; TODO: Move back in time by filtering on txd (transaction's utc date) -;; TODO: Add some memoization with swark.core/memoire -;; TODO: Implement joins via join-rows, always one to many? - -(defn- utc-now [] - #?(:cljs (.toUTCIsoString (gd/DateTime.)) - :clj (.toString (Instant/now)))) - -;; NOTE: These serializers are extensible, define your own methods for your database. -(defmulti value-serializer (juxt ::primary-key ::attribute)) -(defmethod value-serializer :default [_] swark/->str) - -(defmulti attribute-serializer ::primary-key) -(defmethod attribute-serializer :default [_] swark/->str) - -(defmulti primary-value-serializer ::primary-key) -(defmethod primary-value-serializer :default [_] swark/->str) - -(defmulti primary-key-serializer ::primary-key) -(defmethod primary-key-serializer :default [_] swark/->str) - -(defn- unparse - [row-item] - (-> row-item - (update ::primary-key (primary-key-serializer row-item)) - (update ::primary-value (primary-value-serializer row-item)) - (update ::attribute (attribute-serializer row-item)) - (update ::value (value-serializer row-item)) - (update ::flags #(when (seq %) (->> % (map swark/->str) (into []) str))))) - -(def ^:private entry->row (juxt ::tx-utc-at ::primary-key ::primary-value ::attribute ::value ::flags)) - -(defn- serialize - [{:keys [primary-key flags] - :or {flags #{}}} item] - (let [entity (find item primary-key) - tx-utc-at (utc-now) - archived? (some-> flags ::archived) - item' (if archived? - (select-keys item [primary-key]) - (dissoc item primary-key))] - (assert entity) - (->> item' - (map (fn [[attribute value]] - {::tx-utc-at tx-utc-at - ::entity entity - ::primary-key primary-key - ::primary-value (get item primary-key) - ::attribute attribute - ::value value - ::flags (map name flags)})) - (map unparse) - (map entry->row)))) - -(defmulti value-parser (juxt ::primary-key ::attribute)) -(defmethod value-parser :default [_] identity) - -(defmulti attribute-parser ::primary-key) -(defmethod attribute-parser :default [_] keyword) - -(defmulti primary-value-parser ::primary-key) -(defmethod primary-value-parser :default [_] identity) - -(defmulti primary-key-parser ::primary-key) -(defmethod primary-key-parser :default [_] keyword) - -(def ^:private row->entry - (partial zipmap [::tx-utc-at ::primary-key ::primary-value ::attribute ::value ::flags])) - -(defn- parse-flags - [s] - (let [coll (edn/read-string s)] - (when (coll? coll) - (->> coll - (mapv (partial keyword (namespace ::this))) - set)))) - -(defn- parse-entry [entry] - (let [find-entity (juxt ::primary-key ::primary-value) - entry (as-> entry e - (update e ::primary-key (primary-key-parser e #_ntry)) - (update e ::primary-value (primary-value-parser e #_ntry)) - (update e ::attribute (attribute-parser e #_ntry)) - (update e ::value (value-parser e #_ntry)) - (update e ::flags parse-flags)) - entity (find-entity entry)] - (assoc entry ::entity entity))) - -(defn- entry->map [{::keys [attribute value entity flags] :as entry}] - {entity (with-meta (into {attribute value} [entity]) {::flags flags}) #_entry}) - -(defn- merge-entries [map1 {::keys [attribute] :as map2}] - (cond - (some-> map2 meta ::flags ::archived) - nil ; Return nil, this value is to be removed from the result later - - (some-> map2 meta ::flags ::deleted) - (dissoc map1 attribute) - - :else - (merge map1 map2))) - -(defn- filter-entry [props entry] - (if-not (seq props) - true - (let [props (->> props (filter (comp ifn? val)) (into {})) - keyseq (keys props) - keys (set/intersection (-> entry keys set) (-> props keys set)) - values (if (seq keys) - (apply juxt keys) - (fn [_] (vector nil)))] - (some->> (select-keys entry keyseq) - (merge-with #(%1 %2) props) - values - seq - (every? identity))))) - -(defn- filterer [props] - (filter (partial filter-entry props))) - -(defn- merge-rows - "Return eagerly parsed and merged rows" - ([rows] - (merge-rows nil rows)) - ;; TODO: Make it possible to return only the items from the last tx. Or a specific tx? - ([{:keys [post-merge-parser where] - :or {post-merge-parser identity - where identity} - :as props} rows] - (keep - (comp #(when (where %) %) post-merge-parser val) ; Only keep vals of the db maps, and omit archived entries - (transduce - (comp - (map row->entry) - (map parse-entry) - (filterer props) - (map entry->map)) - (partial merge-with merge-entries) - rows)))) - -(defn- diff [primary-key & items] - (let [entity (find (apply merge items) primary-key) - [removed added _] (apply data/diff items) - removed (some->> removed - (remove (comp (or added {}) key)) - seq - (into {}))] - {::added added - ::removed removed})) - -(defn- diff-rows [{:keys [primary-key] :as props} & items] - (let [entity (select-keys (apply merge items) [primary-key]) - {::keys - [added removed]} (apply diff primary-key items)] - (concat - (serialize (assoc props :flags #{::deleted}) (merge removed entity)) - (serialize props (merge added entity))))) - -(defn- upsert-rows - [db-items {:keys [primary-key next-primary-val] - :or {next-primary-val swark/unid} - :as props} item] - (let [update? (contains? item primary-key) - next-pval #(->> db-items - (map (fn [item] (get item primary-key))) ; NOTE: primary-key doesn't have to be a keyword! - set - next-primary-val) - item (cond-> item - (not update?) (assoc primary-key (next-pval))) - entity (find item primary-key)] - (if update? - (diff-rows props (->> db-items (filter (comp #{entity} #(find % primary-key))) first) item) - (serialize props item)))) - -(defn- upsert - [rows {:keys [pre-upsert-serializer primary-key] - :or {pre-upsert-serializer identity} - :as props} & items] - (assert (seq items)) - (let [primary-values (fn [items] (->> items (map #(get % primary-key)) set)) - db-items (merge-rows {::primary-key #{primary-key}} rows)] - (reduce - (fn [new-rows item] - ;; Take new rows from the other upserted items into account as well (for next-primary-val fn) - (let [db-items' (concat db-items (merge-rows {} new-rows))] - (concat new-rows (upsert-rows db-items' props item)))) - nil - (map pre-upsert-serializer items)))) - -(defn- archive - [rows {:keys [primary-key] :as props} & items] - (assert (and (seq items) (every? #(get % primary-key) items))) - (mapcat (partial serialize (assoc props :flags #{::archived})) items)) - -;; Instead of CRUD, we have URA -;; Upsert = Create and Update -;; Read = Read -;; Archive = Delete - -(defprotocol Cedric - (upsert-items [this props items] "Creates or updates the items, returning the items (with primary-key if created).") - (find-by-entity [this entity] "Finds and returns one db entry, based off one specific entity.") - (find-by-primary-key [this predicate props] "Finds and returns db entries where primary-key matches predicate, with additional props like {:where (comp #{'aname'} :name)}") - (read-items [this props] "Returns all items with low level selectors in props, like ::primary-key, ::primary-value, ::entity, ::attribute, ::value and :where.") - (archive-items [this props items] "Marks the items as archived in the db. Returns {::archived 2} where 2 is the item count actually archived.")) - -(defrecord Mem [rows-atom] - Cedric - (upsert-items [this {:keys [primary-key] :as props} items] - ;; TODO: Return only the items from this (the last) tx - (let [updated-pvals (seq (keep #(get % primary-key) items))] - (->> (swap! rows-atom (fn [rows] (concat rows (apply upsert rows props items)))) - (merge-rows (cond-> {::primary-key #{primary-key}} - updated-pvals (assoc ::primary-value (set updated-pvals))))))) - (find-by-entity [this entity] - (-> this (read-items {::entity #{entity}}) first)) - (find-by-primary-key [this predicate props] - (-> this (read-items (merge props {::primary-key predicate})))) - (read-items [this props] (merge-rows props @rows-atom)) - (archive-items [this {:keys [primary-key] :as props} items] - (swap! rows-atom (fn [rows] (concat rows (apply archive rows props items)))) - (count items))) - -(defn- write-csv! [filename rows] - (with-open [writer (io/writer filename :append true)] - (csv/write-csv writer rows :separator \;))) - -(defn- open-or-create! [filename] - (loop [reader (swark/jab io/reader filename) - retries-left 3] - (if (or reader (zero? retries-left)) - reader - (do - (write-csv! filename []) - (recur (swark/jab io/reader filename) (dec retries-left)))))) - -(defn- read-csv [filename] - (with-open [reader (open-or-create! filename)] - (-> reader (csv/read-csv :separator \;) doall))) - -#?(:clj - (defrecord Csv [filename] - Cedric - (upsert-items [this {:keys [primary-key] :as props} items] - (let [rows (read-csv filename) - new-rows (apply upsert rows props items) - updated-pvals (seq (keep #(get % primary-key) items))] - (write-csv! filename new-rows) - (merge-rows - (cond-> {::primary-key #{primary-key}} - updated-pvals (assoc ::primary-value (set updated-pvals))) - new-rows))) - (find-by-entity [this entity] - (-> this (read-items {::entity #{entity}}) first)) - (find-by-primary-key [this predicate props] - (-> this (read-items (merge props {::primary-key predicate})))) - (read-items [this props] (merge-rows props (read-csv filename))) - (archive-items [this props items] - (let [rows (read-csv filename) - new-rows (apply archive rows props items)] - (write-csv! filename new-rows) - (count new-rows))))) - -(defn make-connection - "Returns a map with ::transact! and ::close! functions." - [db] - (let [conn (atomic/atomic db)] - {::transact! (partial atomic/put! conn) - ::close! #(atomic/close! conn)})) - -(comment - ;; TODO: have a look a the doto macro in core... - (let [connection (-> "/tmp/testdb123.csv" Csv. make-connection)] - (def transact! (::transact! connection)) ; Define transact! for this connection - (def close! (::close! connection))) ; Define close! for this connection - - ;; Upsert items via the transact! function - (transact! upsert-items {:primary-key :user/id} [{:user/name "Arnold"} {:user/name "Naomi"} {:user/name "Theodor"}]) - ;; Read (all) items via the transact! function - (transact! read-items {}) - ;; Archive an item via the transact! function - (transact! archive-items {:primary-key :user/id} [{:user/id "4"}]) - ;; Close the connection via the close! function - (close!)) diff --git a/test/swark/cedric_test.clj b/test/swark/cedric_test.clj deleted file mode 100644 index 2f1b380..0000000 --- a/test/swark/cedric_test.clj +++ /dev/null @@ -1,83 +0,0 @@ -(ns swark.cedric-test - (:require [clojure.edn :as edn] - [clojure.test :refer [are deftest is testing]] - [swark.cedric :as sut] - [swark.core :as swark]) - (:import [swark.cedric Mem Csv])) - -;; Parse :id record's category value as clojure object (int) -(remove-method sut/value-parser [:id :category]) -(defmethod sut/value-parser [:id :category] [_] - edn/read-string) - -;; Parse user record's gender value as a keyword -(remove-method sut/value-parser [::user-id :user/gender]) -(defmethod sut/value-parser [::user-id :user/gender] [_] - keyword) - -;; Parse primary-value as clojure object (int) -(remove-method sut/primary-value-parser ::user-id) -(remove-method sut/primary-value-parser :id) -(defmethod sut/primary-value-parser ::user-id [_] - edn/read-string) -(defmethod sut/primary-value-parser :id [_] - edn/read-string) - -(deftest pipeline-test - (let [user1 {:id 1 :username "Stan" :category 1} - user2 {::user-id 2 :user/name "Nats" :user/gender :unknown} - rows1 (#'sut/serialize {:primary-key :id} user1) - rows2 (#'sut/serialize {:primary-key ::user-id} user2)] - (are [result rows] (= result (->> rows - (map (partial drop 1)) - (map (partial take 4)))) - [["id" "1" "username" "Stan"] - ["id" "1" "category" "1"]] rows1 - [["swark.cedric-test/user-id" "2" "user/name" "Nats"] - ["swark.cedric-test/user-id" "2" "user/gender" "unknown"]] rows2) - (is - (= [{:username "Stan" :id 1 :category 1} - {:user/name "Nats" ::user-id 2 :user/gender :unknown}] - (#'sut/merge-rows (concat rows1 rows2)))))) - -(def ^:private NAMES #{"Alfa" "Bravo" "Charlie" "Delta" "Echo" "Foxtrot" "Golf" "Hotel" "India" "Juliett" "Kilo" "Lima" "Mike" "November" "Oscar" "Papa" "Quebec" "Romeo" "Sierra" "Tango" "Uniform" "Victor" "Whiskey" "X-ray" "Yankee" "Zulu"}) - -(defn- some-names - ([] - (some-names 2)) - ([n] - (assert (< n 26)) - (->> NAMES shuffle (take n)))) - -(deftest implementation - ;; Test all implementations in exactly the same way! - (doseq [make-db [#(Mem. (atom nil)) - #(Csv. (str "/tmp/testdb-" (swark/unid) ".csv"))]] - (let [{::sut/keys - [transact! close!]} (sut/make-connection (make-db)) - props {:primary-key :person/id} - the-names (some-names 25) - persons (map (partial assoc nil :person/name) the-names) - result (transact! sut/upsert-items props persons)] - (testing "upsert-items" - (testing "returns the upserted items" - (is (-> result count (= 25))) - (is (->> result (map :person/name) set (= (set the-names)))))) - (let [new-names (some-names 3) - persons (->> result - shuffle - (take 3) - (map #(assoc %2 :person/name %1) new-names)) - updated (transact! sut/upsert-items props persons)] - (testing "returns the updated items" - (is (-> updated count (= 3))) - (is (->> updated (map :person/name) set (= (set new-names)))))) - (let [persons (->> result - shuffle - (take 5)) - archived (transact! sut/archive-items props persons)] - (testing "returns the number of ::archived items" - (is (= 5 archived)))) - (testing "returns all the items" - (is (-> (transact! sut/read-items {}) count #{20}))) - (close!))))