Migrate from old prescience-server project

.gitignore

+elasticsearch.admin.password

README.md

+# elasticsearch
+
+The backend database for Prescience based on Elasticsearch.

elasticsearch-mappings/prescience/fluid.json

+{
+    "fluid" : {
+        "properties" : {
+            "procId" : { "type" : "string", "index" : "not_analyzed" },
+            "time" : { "type" : "date", "format" : "yyyy-MM-dd HH:mm:ss.SSS" },
+            "name" : { "type" : "string", "index" : "not_analyzed" },
+            "value" : { "type" : "float" },
+            "units" : { "type" : "string", "index" : "not_analyzed" }
+        }
+    }
+}

elasticsearch-mappings/prescience/lab.json

+{
+    "lab" : {
+        "properties" : {
+            "procId" : { "type" : "string", "index" : "not_analyzed" },
+            "time" : { "type" : "date", "format" : "yyyy-MM-dd HH:mm:ss.SSS" },
+            "name" : { "type" : "string", "index" : "not_analyzed" },
+            "numericValue" : { "type" : "float" },
+            "textValue" : { "type" : "string", "index" : "not_analyzed" }
+        }
+    }
+}

elasticsearch-mappings/prescience/medication.json

+{
+    "medication" : {
+        "properties" : {
+            "procId" : { "type" : "string", "index" : "not_analyzed" },
+            "startTime" : { "type" : "date", "format" : "yyyy-MM-dd HH:mm:ss.SSS" },
+            "endTime" : { "type" : "date", "format" : "yyyy-MM-dd HH:mm:ss.SSS" },
+            "name" : { "type" : "string", "index" : "not_analyzed" },
+            "value" : { "type" : "float" },
+            "units" : { "type" : "string", "index" : "not_analyzed" }
+        }
+    }
+}

elasticsearch-mappings/prescience/monitor.json

+{
+    "monitor" : {
+        "properties" : {
+            "procId" : { "type" : "string", "index" : "not_analyzed" },
+            "time" : { "type" : "date", "format" : "yyyy-MM-dd HH:mm:ss.SSS" },
+            "sensor" : { "type" : "string", "index" : "not_analyzed" },
+            "numericValue" : { "type" : "float" },
+            "textValue" : { "type" : "string", "index" : "not_analyzed" }
+        }
+    }
+}

elasticsearch-mappings/prescience/orcapreop.json

+{
+    "orcapreop" : {
+        "properties" : {
+            "procId" : { "type" : "string", "index" : "not_analyzed" },
+            "time" : { "type" : "date", "format" : "yyyy-MM-dd HH:mm:ss.SSS" },
+            "reviewTitle" : { "type" : "string" },
+            "description" : { "type" : "string" }
+        }
+    }
+}

elasticsearch-mappings/prescience/recorded-score.json

+{
+    "recorded-score" : {
+        "properties" : {
+            "@timestamp" : { "type" : "date", "format" : "yyyy-MM-dd HH:mm:ss.SSS" },
+            "user" : { "type" : "string", "index" : "not_analyzed" },
+            "score" : { "type" : "float" },
+            "testCaseIndex" : { "type" : "int" },
+            "testCaseTimeIndex" : { "type" : "int" },
+            "time" : { "type" : "date", "format" : "yyyy-MM-dd HH:mm:ss.SSS" },
+            "procId" : { "type" : "string", "index" : "not_analyzed" },
+            "version" : { "type" : "string", "index" : "not_analyzed" }
+        }
+    }
+}

elasticsearch-mappings/prescience/summaryinfo.json

+{
+    "summaryinfo" : {
+        "properties" : {
+            "procId" : { "type" : "string", "index" : "not_analyzed" },
+            "date" : { "type" : "date", "format" : "yyyy-MM-dd" },
+            "age" : { "type" : "float" },
+            "gender" : { "type" : "string", "index" : "not_analyzed" },
+            "asaCode" : { "type" : "string", "index" : "not_analyzed" },
+            "heightInches" : { "type" : "float" },
+            "weightPounds" : { "type" : "float" },
+            "procText" : { "type" : "string" },
+            "cptCode" : { "type" : "string", "index" : "not_analyzed" },
+            "anesthesiaType" : { "type" : "string", "index" : "not_analyzed" },
+            "surgDiagnosis" : { "type" : "string" },
+            "orLocation" : { "type" : "string", "index" : "not_analyzed" },
+            "anesthesiaStartTime" : { "type" : "date", "format" : "yyyy-MM-dd HH:mm:ss.SSS" },
+            "inRoomTime" : { "type" : "date", "format" : "yyyy-MM-dd HH:mm:ss.SSS" },
+            "anesthesiaReadyTime" : { "type" : "date", "format" : "yyyy-MM-dd HH:mm:ss.SSS" },
+            "procStartTime" : { "type" : "date", "format" : "yyyy-MM-dd HH:mm:ss.SSS" },
+            "procEndTime" : { "type" : "date", "format" : "yyyy-MM-dd HH:mm:ss.SSS" },
+            "leaveOrTime" : { "type" : "date", "format" : "yyyy-MM-dd HH:mm:ss.SSS" },
+            "anesthesiaEndTime" : { "type" : "date", "format" : "yyyy-MM-dd HH:mm:ss.SSS" }
+        }
+    }
+}

elasticsearch-mappings/prescience/user-state.json

+{
+    "userstate" : {
+        "properties" : {
+            "@timestamp" : { "type" : "date", "format" : "yyyy-MM-dd HH:mm:ss.SSS" },
+            "testCaseIndex" : { "type" : "int" },
+            "testCaseTimeIndex" : { "type" : "int" }
+        }
+    }
+}

logstash.config

+input {
+	stdin {}
+}
+
+filter {
+	grok {
+		match => {
+			"message" => "%{WORD:type},.*"
+		}
+	}
+
+	if [type] == "summaryinfo" {
+		grok {
+			match => {
+				"message" => "summaryinfo,%{WORD:procId},(?<date>[^\,]+),(?<age>[^\,]+),(?<gender>[^\,]+),(?<asaCode>[^\,]+),(?<heightInches>[^\,]+),(?<weightPounds>[^\,]+),(?<procText>[^\,]+),(?<cptCode>[^\,]+),(?<anesthesiaType>[^\,]+),(?<surgDiagnosis>[^\,]+),(?<orLocation>[^\,]+),(?<anesthesiaStartTime>[^\,]+),(?<inRoomTime>[^\,]+),(?<anesthesiaReadyTime>[^\,]+),(?<procStartTime>[^\,]+),(?<procEndTime>[^\,]+),(?<leaveOrTime>[^\,]+),(?<anesthesiaEndTime>[^\,]+)"
+	        }
+	    }
+		mutate { add_field => { "_id" => "%{type}-%{procId}" } }
+
+	    date {
+	        timezone => "America/Los_Angeles"
+	        match => ["anesthesiaStartTime", "ISO8601"]
+	    }
+
+	} else if [type] == "monitor" {
+		grok {
+			match => {
+				"message" => "monitor,%{WORD:procId},%{TIMESTAMP_ISO8601:time},%{WORD:sensor},(?<numericValue>[0-9\.]+),%{WORD:textValue}"
+	        }
+	    }
+		mutate { add_field => { "_id" => "%{type}-%{procId}-%{time}-%{sensor}" } }
+
+	    if [numericValue] != "NULL" {
+			mutate { convert => { "numericValue" => "float" } }
+	    }
+
+	    date {
+	        timezone => "America/Los_Angeles"
+	        match => ["time", "ISO8601"]
+	    }
+
+	} else if [type] == "medication" {
+		grok {
+			match => {
+				"message" => "medication,%{WORD:procId},%{TIMESTAMP_ISO8601:startTime},%{WORD:endTime},%{WORD:name},%{NUMBER:value},%{WORD:units}"
+	        }
+	    }
+		mutate { add_field => { "_id" => "%{type}-%{procId}-%{startTime}-%{name}-%{value}" } }
+
+	    mutate { convert => { "value" => "float" } }
+
+	    date {
+	        timezone => "America/Los_Angeles"
+	        match => ["startTime", "ISO8601"]
+	    }
+
+
+	} else if [type] == "fluid" {
+		grok {
+			match => {
+				"message" => "fluid,%{WORD:procId},%{TIMESTAMP_ISO8601:time},%{NUMBER:value},(?<name>[^\,]+),%{WORD:units}"
+	        }
+	    }
+		mutate { add_field => { "_id" => "%{type}-%{procId}-%{time}-%{name}-%{value}" } }
+
+	    mutate { convert => { "value" => "float" } }
+
+	    date {
+	        timezone => "America/Los_Angeles"
+	        match => ["time", "ISO8601"]
+	    }
+
+
+	} else if [type] == "lab" {
+		grok {
+			match => {
+				"message" => "lab,%{WORD:procId},%{TIMESTAMP_ISO8601:time},(?<name>[^\,]+),(?<numericValue>[^\,]+),(?<textValue>[^\,]+)"
+	        }
+	    }
+		mutate { add_field => { "_id" => "%{type}-%{procId}-%{time}-%{name}" } }
+
+	    if [numericValue] != "NULL" {
+			mutate { convert => { "numericValue" => "float" } }
+	    }
+
+	    date {
+	        timezone => "America/Los_Angeles"
+	        match => ["time", "ISO8601"]
+	    }
+
+	} else if [type] == "orcapreop" {
+		grok {
+			match => {
+				"message" => "orcapreop,%{WORD:procId},%{TIMESTAMP_ISO8601:time},(?<reviewTitle>[^\,]+),(?<description>[^\,]+)"
+	        }
+	    }
+		mutate { add_field => { "_id" => "%{type}-%{procId}-%{time}-%{reviewTitle}-%{description}" } }
+
+	    date {
+	        timezone => "America/Los_Angeles"
+	        match => ["time", "ISO8601"]
+	    }
+	} else if [type] == "event_10_5_5" {
+		grok {
+			match => {
+				"message" => "event_10_5_5,%{WORD:procId},%{TIMESTAMP_ISO8601:time},(?<label>[^\,]+)"
+	        }
+	    }
+
+	    date {
+	        timezone => "America/Los_Angeles"
+	        match => ["time", "ISO8601"]
+	    }
+
+		mutate { add_field => { "_id" => "%{type}-%{procId}-%{@timestamp}" } }
+	}
+
+	ruby {
+        code => "
+            hash = event.to_hash
+            hash.each do |k,v|
+                if v == 'NULL'
+                    event.remove(k)
+                end
+            end
+        "
+    }
+
+	mutate { remove_field => "message" }
+	#mutate { add_field => { "learningLabel" => "training"} }
+}
+
+# output {
+#     stdout { codec => rubydebug }
+#     # elasticsearch {
+#     # 	protocol => "http"
+#     # 	user => "admin"
+#     # 	#password => "REAL_PASS_HERE"
+#     # 	index => "prescience"
+#     # 	document_type => "%{type}"
+#     # }
+# }

run-logstash.sh

+#! /bin/bash
+
+# our first arg should be either "train" or "validation"
+DATA_DIR=/projects/leelab/data/ML_OR/data/merged/merged.raw$1/
+
+# load the password from disk
+PASS=`cat elasticsearch.admin.password`
+
+# build the dynamic parts of the logstash configuration
+# (these are the parts depending on the password and train vs. validation)
+read -d '' CONFIG <<EOF
+filter {
+    mutate { add_field => { "learningLabel" => "$1" } }
+}
+output {
+    elasticsearch {
+        protocol => "http"
+        user => "admin"
+        index => "prescience"
+        password => "$PASS"
+        document_type => "%{type}"
+        document_id => "%{_id}"
+    }
+}
+EOF
+
+# run logstash
+find $DATA_DIR -type f | xargs -n 10 cat | logstash/bin/logstash --config logstash.config -e "$CONFIG"