Skip to content
Snippets Groups Projects
Commit cde7db43 authored by Scott Lundberg's avatar Scott Lundberg
Browse files

Migrate from old prescience-server project

parents
No related branches found
No related tags found
No related merge requests found
elasticsearch.admin.password
# elasticsearch
The backend database for Prescience based on Elasticsearch.
{
"fluid" : {
"properties" : {
"procId" : { "type" : "string", "index" : "not_analyzed" },
"time" : { "type" : "date", "format" : "yyyy-MM-dd HH:mm:ss.SSS" },
"name" : { "type" : "string", "index" : "not_analyzed" },
"value" : { "type" : "float" },
"units" : { "type" : "string", "index" : "not_analyzed" }
}
}
}
{
"lab" : {
"properties" : {
"procId" : { "type" : "string", "index" : "not_analyzed" },
"time" : { "type" : "date", "format" : "yyyy-MM-dd HH:mm:ss.SSS" },
"name" : { "type" : "string", "index" : "not_analyzed" },
"numericValue" : { "type" : "float" },
"textValue" : { "type" : "string", "index" : "not_analyzed" }
}
}
}
{
"medication" : {
"properties" : {
"procId" : { "type" : "string", "index" : "not_analyzed" },
"startTime" : { "type" : "date", "format" : "yyyy-MM-dd HH:mm:ss.SSS" },
"endTime" : { "type" : "date", "format" : "yyyy-MM-dd HH:mm:ss.SSS" },
"name" : { "type" : "string", "index" : "not_analyzed" },
"value" : { "type" : "float" },
"units" : { "type" : "string", "index" : "not_analyzed" }
}
}
}
{
"monitor" : {
"properties" : {
"procId" : { "type" : "string", "index" : "not_analyzed" },
"time" : { "type" : "date", "format" : "yyyy-MM-dd HH:mm:ss.SSS" },
"sensor" : { "type" : "string", "index" : "not_analyzed" },
"numericValue" : { "type" : "float" },
"textValue" : { "type" : "string", "index" : "not_analyzed" }
}
}
}
{
"orcapreop" : {
"properties" : {
"procId" : { "type" : "string", "index" : "not_analyzed" },
"time" : { "type" : "date", "format" : "yyyy-MM-dd HH:mm:ss.SSS" },
"reviewTitle" : { "type" : "string" },
"description" : { "type" : "string" }
}
}
}
{
"recorded-score" : {
"properties" : {
"@timestamp" : { "type" : "date", "format" : "yyyy-MM-dd HH:mm:ss.SSS" },
"user" : { "type" : "string", "index" : "not_analyzed" },
"score" : { "type" : "float" },
"testCaseIndex" : { "type" : "int" },
"testCaseTimeIndex" : { "type" : "int" },
"time" : { "type" : "date", "format" : "yyyy-MM-dd HH:mm:ss.SSS" },
"procId" : { "type" : "string", "index" : "not_analyzed" },
"version" : { "type" : "string", "index" : "not_analyzed" }
}
}
}
{
"summaryinfo" : {
"properties" : {
"procId" : { "type" : "string", "index" : "not_analyzed" },
"date" : { "type" : "date", "format" : "yyyy-MM-dd" },
"age" : { "type" : "float" },
"gender" : { "type" : "string", "index" : "not_analyzed" },
"asaCode" : { "type" : "string", "index" : "not_analyzed" },
"heightInches" : { "type" : "float" },
"weightPounds" : { "type" : "float" },
"procText" : { "type" : "string" },
"cptCode" : { "type" : "string", "index" : "not_analyzed" },
"anesthesiaType" : { "type" : "string", "index" : "not_analyzed" },
"surgDiagnosis" : { "type" : "string" },
"orLocation" : { "type" : "string", "index" : "not_analyzed" },
"anesthesiaStartTime" : { "type" : "date", "format" : "yyyy-MM-dd HH:mm:ss.SSS" },
"inRoomTime" : { "type" : "date", "format" : "yyyy-MM-dd HH:mm:ss.SSS" },
"anesthesiaReadyTime" : { "type" : "date", "format" : "yyyy-MM-dd HH:mm:ss.SSS" },
"procStartTime" : { "type" : "date", "format" : "yyyy-MM-dd HH:mm:ss.SSS" },
"procEndTime" : { "type" : "date", "format" : "yyyy-MM-dd HH:mm:ss.SSS" },
"leaveOrTime" : { "type" : "date", "format" : "yyyy-MM-dd HH:mm:ss.SSS" },
"anesthesiaEndTime" : { "type" : "date", "format" : "yyyy-MM-dd HH:mm:ss.SSS" }
}
}
}
{
"userstate" : {
"properties" : {
"@timestamp" : { "type" : "date", "format" : "yyyy-MM-dd HH:mm:ss.SSS" },
"testCaseIndex" : { "type" : "int" },
"testCaseTimeIndex" : { "type" : "int" }
}
}
}
input {
stdin {}
}
filter {
grok {
match => {
"message" => "%{WORD:type},.*"
}
}
if [type] == "summaryinfo" {
grok {
match => {
"message" => "summaryinfo,%{WORD:procId},(?<date>[^\,]+),(?<age>[^\,]+),(?<gender>[^\,]+),(?<asaCode>[^\,]+),(?<heightInches>[^\,]+),(?<weightPounds>[^\,]+),(?<procText>[^\,]+),(?<cptCode>[^\,]+),(?<anesthesiaType>[^\,]+),(?<surgDiagnosis>[^\,]+),(?<orLocation>[^\,]+),(?<anesthesiaStartTime>[^\,]+),(?<inRoomTime>[^\,]+),(?<anesthesiaReadyTime>[^\,]+),(?<procStartTime>[^\,]+),(?<procEndTime>[^\,]+),(?<leaveOrTime>[^\,]+),(?<anesthesiaEndTime>[^\,]+)"
}
}
mutate { add_field => { "_id" => "%{type}-%{procId}" } }
date {
timezone => "America/Los_Angeles"
match => ["anesthesiaStartTime", "ISO8601"]
}
} else if [type] == "monitor" {
grok {
match => {
"message" => "monitor,%{WORD:procId},%{TIMESTAMP_ISO8601:time},%{WORD:sensor},(?<numericValue>[0-9\.]+),%{WORD:textValue}"
}
}
mutate { add_field => { "_id" => "%{type}-%{procId}-%{time}-%{sensor}" } }
if [numericValue] != "NULL" {
mutate { convert => { "numericValue" => "float" } }
}
date {
timezone => "America/Los_Angeles"
match => ["time", "ISO8601"]
}
} else if [type] == "medication" {
grok {
match => {
"message" => "medication,%{WORD:procId},%{TIMESTAMP_ISO8601:startTime},%{WORD:endTime},%{WORD:name},%{NUMBER:value},%{WORD:units}"
}
}
mutate { add_field => { "_id" => "%{type}-%{procId}-%{startTime}-%{name}-%{value}" } }
mutate { convert => { "value" => "float" } }
date {
timezone => "America/Los_Angeles"
match => ["startTime", "ISO8601"]
}
} else if [type] == "fluid" {
grok {
match => {
"message" => "fluid,%{WORD:procId},%{TIMESTAMP_ISO8601:time},%{NUMBER:value},(?<name>[^\,]+),%{WORD:units}"
}
}
mutate { add_field => { "_id" => "%{type}-%{procId}-%{time}-%{name}-%{value}" } }
mutate { convert => { "value" => "float" } }
date {
timezone => "America/Los_Angeles"
match => ["time", "ISO8601"]
}
} else if [type] == "lab" {
grok {
match => {
"message" => "lab,%{WORD:procId},%{TIMESTAMP_ISO8601:time},(?<name>[^\,]+),(?<numericValue>[^\,]+),(?<textValue>[^\,]+)"
}
}
mutate { add_field => { "_id" => "%{type}-%{procId}-%{time}-%{name}" } }
if [numericValue] != "NULL" {
mutate { convert => { "numericValue" => "float" } }
}
date {
timezone => "America/Los_Angeles"
match => ["time", "ISO8601"]
}
} else if [type] == "orcapreop" {
grok {
match => {
"message" => "orcapreop,%{WORD:procId},%{TIMESTAMP_ISO8601:time},(?<reviewTitle>[^\,]+),(?<description>[^\,]+)"
}
}
mutate { add_field => { "_id" => "%{type}-%{procId}-%{time}-%{reviewTitle}-%{description}" } }
date {
timezone => "America/Los_Angeles"
match => ["time", "ISO8601"]
}
} else if [type] == "event_10_5_5" {
grok {
match => {
"message" => "event_10_5_5,%{WORD:procId},%{TIMESTAMP_ISO8601:time},(?<label>[^\,]+)"
}
}
date {
timezone => "America/Los_Angeles"
match => ["time", "ISO8601"]
}
mutate { add_field => { "_id" => "%{type}-%{procId}-%{@timestamp}" } }
}
ruby {
code => "
hash = event.to_hash
hash.each do |k,v|
if v == 'NULL'
event.remove(k)
end
end
"
}
mutate { remove_field => "message" }
#mutate { add_field => { "learningLabel" => "training"} }
}
# output {
# stdout { codec => rubydebug }
# # elasticsearch {
# # protocol => "http"
# # user => "admin"
# # #password => "REAL_PASS_HERE"
# # index => "prescience"
# # document_type => "%{type}"
# # }
# }
#! /bin/bash
# our first arg should be either "train" or "validation"
DATA_DIR=/projects/leelab/data/ML_OR/data/merged/merged.raw$1/
# load the password from disk
PASS=`cat elasticsearch.admin.password`
# build the dynamic parts of the logstash configuration
# (these are the parts depending on the password and train vs. validation)
read -d '' CONFIG <<EOF
filter {
mutate { add_field => { "learningLabel" => "$1" } }
}
output {
elasticsearch {
protocol => "http"
user => "admin"
index => "prescience"
password => "$PASS"
document_type => "%{type}"
document_id => "%{_id}"
}
}
EOF
# run logstash
find $DATA_DIR -type f | xargs -n 10 cat | logstash/bin/logstash --config logstash.config -e "$CONFIG"
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment