Skip to content
Snippets Groups Projects
Commit a9bc84c4 authored by bigbes's avatar bigbes
Browse files

Fix codestyle in measurements section of code 

(trailing whitespaces, 2 spaces instead of tabs and e.t.c)
parent 2d6185ee
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
Showing
with 831 additions and 818 deletions
core/src/main/java/com/yahoo/ycsb/measurements/Measurements.java
+ 240
− 234
View file @ a9bc84c4
/**
* Copyright (c) 2010 Yahoo! Inc. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you
* may not use this file except in compliance with the License. You
* may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
* implied. See the License for the specific language governing
* permissions and limitations under the License. See accompanying
* LICENSE file.
/**
* Copyright (c) 2010 Yahoo! Inc. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you
* may not use this file except in compliance with the License. You
* may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
* implied. See the License for the specific language governing
* permissions and limitations under the License. See accompanying
* LICENSE file.
*/
package com.yahoo.ycsb.measurements;
......@@ -25,253 +25,259 @@ import com.yahoo.ycsb.measurements.exporter.MeasurementsExporter;
/**
* Collects latency measurements, and reports them when requested.
*
*
* @author cooperb
*
*/
public class Measurements
{
public static final String MEASUREMENT_TYPE_PROPERTY = "measurementtype";
private static final String MEASUREMENT_TYPE_PROPERTY_DEFAULT = "histogram";
public static final String MEASUREMENT_INTERVAL = "measurement.interval";
private static final String MEASUREMENT_INTERVAL_DEFAULT = "op";
public static final String MEASUREMENT_TYPE_PROPERTY = "measurementtype";
static Measurements singleton=null;
static Properties measurementproperties=null;
public static void setProperties(Properties props)
{
measurementproperties=props;
}
private static final String MEASUREMENT_TYPE_PROPERTY_DEFAULT = "histogram";
public static final String MEASUREMENT_INTERVAL = "measurement.interval";
private static final String MEASUREMENT_INTERVAL_DEFAULT = "op";
/**
* Return the singleton Measurements object.
*/
public synchronized static Measurements getMeasurements()
{
if (singleton==null)
{
singleton=new Measurements(measurementproperties);
}
return singleton;
}
static Measurements singleton=null;
static Properties measurementproperties=null;
final ConcurrentHashMap<String,OneMeasurement> _opToMesurementMap;
final ConcurrentHashMap<String,OneMeasurement> _opToIntendedMesurementMap;
final int _measurementType;
final int _measurementInterval;
private Properties _props;
/**
* Create a new object with the specified properties.
*/
public Measurements(Properties props)
{
_opToMesurementMap=new ConcurrentHashMap<String,OneMeasurement>();
_opToIntendedMesurementMap=new ConcurrentHashMap<String,OneMeasurement>();
_props=props;
String mTypeString = _props.getProperty(MEASUREMENT_TYPE_PROPERTY, MEASUREMENT_TYPE_PROPERTY_DEFAULT);
if (mTypeString.equals("histogram"))
{
_measurementType = 0;
}
else if (mTypeString.equals("hdrhistogram"))
{
_measurementType = 1;
}
else if (mTypeString.equals("hdrhistogram+histogram"))
{
_measurementType = 2;
}
else if (mTypeString.equals("timeseries"))
{
_measurementType = 3;
}
else {
throw new IllegalArgumentException("unknown "+MEASUREMENT_TYPE_PROPERTY+"="+mTypeString);
}
String mIntervalString = _props.getProperty(MEASUREMENT_INTERVAL, MEASUREMENT_INTERVAL_DEFAULT);
if (mIntervalString.equals("op"))
{
_measurementInterval = 0;
}
else if (mIntervalString.equals("intended"))
{
_measurementInterval = 1;
}
else if (mIntervalString.equals("both"))
{
_measurementInterval = 2;
}
else {
throw new IllegalArgumentException("unknown "+MEASUREMENT_INTERVAL+"="+mIntervalString);
}
}
OneMeasurement constructOneMeasurement(String name)
public static void setProperties(Properties props)
{
measurementproperties=props;
}
/**
* Return the singleton Measurements object.
*/
public synchronized static Measurements getMeasurements()
{
if (singleton==null)
{
switch (_measurementType)
{
case 0:
return new OneMeasurementHistogram(name, _props);
case 1:
return new OneMeasurementHdrHistogram(name, _props);
case 2:
return new TwoInOneMeasurement(name,
new OneMeasurementHdrHistogram("Hdr"+name, _props),
new OneMeasurementHistogram("Bucket"+name, _props));
default:
return new OneMeasurementTimeSeries(name, _props);
}
singleton=new Measurements(measurementproperties);
}
return singleton;
}
final ConcurrentHashMap<String,OneMeasurement> _opToMesurementMap;
final ConcurrentHashMap<String,OneMeasurement> _opToIntendedMesurementMap;
final int _measurementType;
final int _measurementInterval;
private Properties _props;
static class StartTimeHolder{
long time;
long startTime(){
if(time == 0) {
return System.nanoTime();
}
else {
return time;
}
}
/**
* Create a new object with the specified properties.
*/
public Measurements(Properties props)
{
_opToMesurementMap=new ConcurrentHashMap<String,OneMeasurement>();
_opToIntendedMesurementMap=new ConcurrentHashMap<String,OneMeasurement>();
_props=props;
String mTypeString = _props.getProperty(MEASUREMENT_TYPE_PROPERTY, MEASUREMENT_TYPE_PROPERTY_DEFAULT);
if (mTypeString.equals("histogram"))
{
_measurementType = 0;
}
else if (mTypeString.equals("hdrhistogram"))
{
_measurementType = 1;
}
else if (mTypeString.equals("hdrhistogram+histogram"))
{
_measurementType = 2;
}
else if (mTypeString.equals("timeseries"))
{
_measurementType = 3;
}
ThreadLocal<StartTimeHolder> tlIntendedStartTime = new ThreadLocal<Measurements.StartTimeHolder>(){
protected StartTimeHolder initialValue() {
return new StartTimeHolder();
};
else {
throw new IllegalArgumentException("unknown "+MEASUREMENT_TYPE_PROPERTY+"="+mTypeString);
}
String mIntervalString = _props.getProperty(MEASUREMENT_INTERVAL, MEASUREMENT_INTERVAL_DEFAULT);
if (mIntervalString.equals("op"))
{
_measurementInterval = 0;
}
else if (mIntervalString.equals("intended"))
{
_measurementInterval = 1;
}
else if (mIntervalString.equals("both"))
{
_measurementInterval = 2;
}
else {
throw new IllegalArgumentException("unknown "+MEASUREMENT_INTERVAL+"="+mIntervalString);
}
}
OneMeasurement constructOneMeasurement(String name)
{
switch (_measurementType)
{
case 0:
return new OneMeasurementHistogram(name, _props);
case 1:
return new OneMeasurementHdrHistogram(name, _props);
case 2:
return new TwoInOneMeasurement(name,
new OneMeasurementHdrHistogram("Hdr"+name, _props),
new OneMeasurementHistogram("Bucket"+name, _props));
default:
return new OneMeasurementTimeSeries(name, _props);
}
}
static class StartTimeHolder {
long time;
long startTime(){
if(time == 0) {
return System.nanoTime();
}
else {
return time;
}
}
}
ThreadLocal<StartTimeHolder> tlIntendedStartTime = new ThreadLocal<Measurements.StartTimeHolder>() {
protected StartTimeHolder initialValue() {
return new StartTimeHolder();
};
public void setIntendedStartTimeNs(long time){
if(_measurementInterval==0)
return;
tlIntendedStartTime.get().time=time;
};
public void setIntendedStartTimeNs(long time) {
if(_measurementInterval==0)
return;
tlIntendedStartTime.get().time=time;
}
public long getIntendedtartTimeNs() {
if(_measurementInterval==0)
return 0L;
return tlIntendedStartTime.get().startTime();
}
/**
* Report a single value of a single metric. E.g. for read latency, operation="READ" and latency is the measured
* value.
*/
public void measure(String operation, int latency)
{
if(_measurementInterval==1)
return;
try
{
OneMeasurement m = getOpMeasurement(operation);
m.measure(latency);
}
public long getIntendedtartTimeNs(){
if(_measurementInterval==0)
return 0L;
return tlIntendedStartTime.get().startTime();
// This seems like a terribly hacky way to cover up for a bug in the measurement code
catch (java.lang.ArrayIndexOutOfBoundsException e)
{
System.out.println("ERROR: java.lang.ArrayIndexOutOfBoundsException - ignoring and continuing");
e.printStackTrace();
e.printStackTrace(System.out);
}
}
/**
* Report a single value of a single metric. E.g. for read latency, operation="READ" and latency is the measured
* value.
*/
public void measure(String operation, int latency)
{
if(_measurementInterval==1)
return;
try
{
OneMeasurement m = getOpMeasurement(operation);
m.measure(latency);
}
// This seems like a terribly hacky way to cover up for a bug in the measurement code
catch (java.lang.ArrayIndexOutOfBoundsException e)
{
System.out.println("ERROR: java.lang.ArrayIndexOutOfBoundsException - ignoring and continuing");
e.printStackTrace();
e.printStackTrace(System.out);
}
}
/**
* Report a single value of a single metric. E.g. for read latency, operation="READ" and latency is the measured
* value.
*/
public void measureIntended(String operation, int latency)
/**
* Report a single value of a single metric. E.g. for read latency, operation="READ" and latency is the measured
* value.
*/
public void measureIntended(String operation, int latency)
{
if(_measurementInterval==0)
return;
try
{
if(_measurementInterval==0)
return;
try
{
OneMeasurement m = getOpIntendedMeasurement(operation);
m.measure(latency);
}
// This seems like a terribly hacky way to cover up for a bug in the measurement code
catch (java.lang.ArrayIndexOutOfBoundsException e)
{
System.out.println("ERROR: java.lang.ArrayIndexOutOfBoundsException - ignoring and continuing");
e.printStackTrace();
e.printStackTrace(System.out);
}
OneMeasurement m = getOpIntendedMeasurement(operation);
m.measure(latency);
}
// This seems like a terribly hacky way to cover up for a bug in the measurement code
catch (java.lang.ArrayIndexOutOfBoundsException e)
{
System.out.println("ERROR: java.lang.ArrayIndexOutOfBoundsException - ignoring and continuing");
e.printStackTrace();
e.printStackTrace(System.out);
}
}
private OneMeasurement getOpMeasurement(String operation) {
OneMeasurement m = _opToMesurementMap.get(operation);
if(m == null)
{
m = constructOneMeasurement(operation);
OneMeasurement oldM = _opToMesurementMap.putIfAbsent(operation, m);
if(oldM != null)
{
m = oldM;
}
}
return m;
private OneMeasurement getOpMeasurement(String operation) {
OneMeasurement m = _opToMesurementMap.get(operation);
if(m == null)
{
m = constructOneMeasurement(operation);
OneMeasurement oldM = _opToMesurementMap.putIfAbsent(operation, m);
if(oldM != null)
{
m = oldM;
}
}
private OneMeasurement getOpIntendedMeasurement(String operation) {
OneMeasurement m = _opToIntendedMesurementMap.get(operation);
if(m == null)
{
final String name = _measurementInterval==1 ? operation : "Intended-" + operation;
m = constructOneMeasurement(name);
OneMeasurement oldM = _opToIntendedMesurementMap.putIfAbsent(operation, m);
if(oldM != null)
{
m = oldM;
}
}
return m;
return m;
}
private OneMeasurement getOpIntendedMeasurement(String operation) {
OneMeasurement m = _opToIntendedMesurementMap.get(operation);
if(m == null)
{
final String name = _measurementInterval==1 ? operation : "Intended-" + operation;
m = constructOneMeasurement(name);
OneMeasurement oldM = _opToIntendedMesurementMap.putIfAbsent(operation, m);
if(oldM != null)
{
m = oldM;
}
}
/**
* Report a return code for a single DB operation.
*/
public void reportReturnCode(String operation, int code)
{
OneMeasurement m = _measurementInterval==1 ?
getOpIntendedMeasurement(operation) :
getOpMeasurement(operation);
m.reportReturnCode(code);
}
return m;
}
/**
* Report a return code for a single DB operation.
*/
public void reportReturnCode(String operation, int code)
{
OneMeasurement m = _measurementInterval==1 ?
getOpIntendedMeasurement(operation) :
getOpMeasurement(operation);
m.reportReturnCode(code);
}
/**
* Export the current measurements to a suitable format.
*
*
* @param exporter Exporter representing the type of format to write to.
* @throws IOException Thrown if the export failed.
*/
public void exportMeasurements(MeasurementsExporter exporter) throws IOException
{
for (OneMeasurement measurement : _opToMesurementMap.values())
{
measurement.exportMeasurements(exporter);
}
for (OneMeasurement measurement : _opToIntendedMesurementMap.values())
{
measurement.exportMeasurements(exporter);
}
for (OneMeasurement measurement : _opToMesurementMap.values())
{
measurement.exportMeasurements(exporter);
}
for (OneMeasurement measurement : _opToIntendedMesurementMap.values())
{
measurement.exportMeasurements(exporter);
}
}
/**
* Return a one line summary of the measurements.
*/
public synchronized String getSummary()
{
String ret="";
for (OneMeasurement m : _opToMesurementMap.values())
{
ret += m.getSummary()+" ";
}
for (OneMeasurement m : _opToIntendedMesurementMap.values())
{
ret += m.getSummary()+" ";
}
return ret;
}
/**
* Return a one line summary of the measurements.
*/
public synchronized String getSummary()
{
String ret="";
for (OneMeasurement m : _opToMesurementMap.values())
{
ret+=m.getSummary()+" ";
}
for (OneMeasurement m : _opToIntendedMesurementMap.values())
{
ret+=m.getSummary()+" ";
}
return ret;
}
}
core/src/main/java/com/yahoo/ycsb/measurements/OneMeasurement.java
+ 76
− 72
View file @ a9bc84c4
/**
* Copyright (c) 2010 Yahoo! Inc. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you
* may not use this file except in compliance with the License. You
* may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
* implied. See the License for the specific language governing
* permissions and limitations under the License. See accompanying
* LICENSE file.
*/
package com.yahoo.ycsb.measurements;
import java.io.IOException;
import com.yahoo.ycsb.measurements.exporter.MeasurementsExporter;
/**
* A single measured metric (such as READ LATENCY)
*/
public abstract class OneMeasurement {
String _name;
final ConcurrentHashMap<Integer, AtomicInteger> returncodes;
public String getName() {
returncodes = new ConcurrentHashMap<Integer, AtomicInteger>();
return _name;
}
/**
* @param _name
*/
public OneMeasurement(String _name) {
this._name = _name;
}
/**
* No need for synchronization, using CHM to deal with that
*/
public void reportReturnCode(int code) {
Integer Icode = code;
AtomicInteger counter = returncodes.get(Icode);
if (counter == null) {
AtomicInteger other = returncodes.putIfAbsent(Icode, counter = new AtomicInteger());
if (other != null) {
counter = other;
}
}
counter.incrementAndGet();
}
public abstract void measure(int latency);
public abstract String getSummary();
/**
* Export the current measurements to a suitable format.
*
* @param exporter Exporter representing the type of format to write to.
* @throws IOException Thrown if the export failed.
*/
public abstract void exportMeasurements(MeasurementsExporter exporter) throws IOException;
}
/**
* Copyright (c) 2010 Yahoo! Inc. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you
* may not use this file except in compliance with the License. You
* may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
* implied. See the License for the specific language governing
* permissions and limitations under the License. See accompanying
* LICENSE file.
*/
package com.yahoo.ycsb.measurements;
import java.io.IOException;
import java.util.concurrent.ConcurrentHashMap;
import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger;
import com.yahoo.ycsb.measurements.exporter.MeasurementsExporter;
/**
* A single measured metric (such as READ LATENCY)
*/
public abstract class OneMeasurement {
String _name;
final ConcurrentHashMap<Integer, AtomicInteger> returncodes;
public String getName() {
return _name;
}
/**
* @param _name
*/
public OneMeasurement(String _name) {
this._name = _name;
this.returncodes = new ConcurrentHashMap<Integer, AtomicInteger>();
}
public abstract void measure(int latency);
public abstract String getSummary();
/**
* No need for synchronization, using CHM to deal with that
*/
public void reportReturnCode(int code) {
Integer Icode = code;
AtomicInteger counter = returncodes.get(Icode);
if (counter == null) {
AtomicInteger other = returncodes.putIfAbsent(Icode, counter = new AtomicInteger());
if (other != null) {
counter = other;
}
}
counter.incrementAndGet();
}
/**
* Export the current measurements to a suitable format.
*
* @param exporter Exporter representing the type of format to write to.
* @throws IOException Thrown if the export failed.
*/
public abstract void exportMeasurements(MeasurementsExporter exporter) throws IOException;
}
core/src/main/java/com/yahoo/ycsb/measurements/OneMeasurementHdrHistogram.java
+ 147
− 146
View file @ a9bc84c4
/**
* Copyright (c) 2010 Yahoo! Inc. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you
* may not use this file except in compliance with the License. You
* may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
* implied. See the License for the specific language governing
* permissions and limitations under the License. See accompanying
* LICENSE file.
*/
package com.yahoo.ycsb.measurements;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.PrintStream;
import java.text.DecimalFormat;
import java.util.Map;
import java.util.Properties;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicInteger;
import org.HdrHistogram.Histogram;
import org.HdrHistogram.HistogramLogWriter;
import org.HdrHistogram.Recorder;
import com.yahoo.ycsb.measurements.exporter.MeasurementsExporter;
/**
* Take measurements and maintain a HdrHistogram of a given metric, such as READ LATENCY.
*
* @author nitsanw
*
*/
public class OneMeasurementHdrHistogram extends OneMeasurement {
// we need one log per measurement histogram
final PrintStream log;
final HistogramLogWriter histogramLogWriter;
final Recorder histogram = new Recorder(3);
Histogram totalHistogram;
public OneMeasurementHdrHistogram(String name, Properties props) {
super(name);
boolean shouldLog = Boolean.parseBoolean(props.getProperty("hdrhistogram.fileoutput", "false"));
if (!shouldLog) {
log = null;
histogramLogWriter = null;
return;
}
try {
final String hdrOutputFilename = props.getProperty("hdrhistogram.output.path", "") +name+".hdr";
log = new PrintStream(new FileOutputStream(hdrOutputFilename), false);
} catch (FileNotFoundException e) {
throw new RuntimeException("Failed to open hdr histogram output file",e);
}
histogramLogWriter = new HistogramLogWriter(log);
histogramLogWriter.outputComment("[Logging for: " + name + "]");
histogramLogWriter.outputLogFormatVersion();
histogramLogWriter.outputStartTime(System.currentTimeMillis());
histogramLogWriter.outputLegend();
}
/**
* It appears latency is reported in micros.
* Using {@link Recorder} to support concurrent updates to histogram.
*
* @see com.yahoo.ycsb.OneMeasurement#measure(int)
*/
public void measure(int latencyInMicros) {
histogram.recordValue(latencyInMicros);
}
/**
* This is called from a main thread, on orderly termination.
*
* @see com.yahoo.ycsb.measurements.OneMeasurement#exportMeasurements(com.yahoo.ycsb.measurements.exporter.MeasurementsExporter)
*/
@Override
public void exportMeasurements(MeasurementsExporter exporter) throws IOException {
// accumulate the last interval which was not caught by status thread
Histogram intervalHistogram = getIntervalHistogramAndAccumulate();
if(histogramLogWriter != null) {
histogramLogWriter.outputIntervalHistogram(intervalHistogram);
// we can close now
log.close();
}
exporter.write(getName(), "Operations", totalHistogram.getTotalCount());
exporter.write(getName(), "AverageLatency(us)", totalHistogram.getMean());
exporter.write(getName(), "MinLatency(us)", totalHistogram.getMinValue());
exporter.write(getName(), "MaxLatency(us)", totalHistogram.getMaxValue());
exporter.write(getName(), "95thPercentileLatency(ms)", totalHistogram.getValueAtPercentile(90)/1000);
exporter.write(getName(), "99thPercentileLatency(ms)", totalHistogram.getValueAtPercentile(99)/1000);
for (Map.Entry<Integer, AtomicInteger> entry : returncodes.entrySet()) {
exporter.write(getName(), "Return=" + entry.getKey(), entry.getValue().get());
}
}
/**
* This is called periodically from the StatusThread. There's a single StatusThread per Client process.
* We optionally serialize the interval to log on this opportunity.
* @see com.yahoo.ycsb.measurements.OneMeasurement#getSummary()
*/
@Override
public String getSummary() {
Histogram intervalHistogram = getIntervalHistogramAndAccumulate();
// we use the summary interval as the histogram file interval.
if(histogramLogWriter != null) {
histogramLogWriter.outputIntervalHistogram(intervalHistogram);
}
DecimalFormat d = new DecimalFormat("#.##");
return "[" + getName() +
": Count=" + intervalHistogram.getTotalCount() +
", Max=" + intervalHistogram.getMaxValue() +
", Min=" + intervalHistogram.getMinValue() +
", Avg=" + d.format(intervalHistogram.getMean()) +
", 90=" + d.format(intervalHistogram.getValueAtPercentile(90)) +
", 99=" + d.format(intervalHistogram.getValueAtPercentile(99)) +
", 99.9=" + d.format(intervalHistogram.getValueAtPercentile(99.9)) +
", 99.99=" + d.format(intervalHistogram.getValueAtPercentile(99.99)) +"]";
}
private Histogram getIntervalHistogramAndAccumulate() {
Histogram intervalHistogram = histogram.getIntervalHistogram();
// add this to the total time histogram.
if (totalHistogram == null) {
totalHistogram = intervalHistogram;
}
else {
totalHistogram.add(intervalHistogram);
}
return intervalHistogram;
}
}
/**
* Copyright (c) 2010 Yahoo! Inc. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you
* may not use this file except in compliance with the License. You
* may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
* implied. See the License for the specific language governing
* permissions and limitations under the License. See accompanying
* LICENSE file.
*/
package com.yahoo.ycsb.measurements;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.PrintStream;
import java.text.DecimalFormat;
import java.util.Map;
import java.util.Properties;
import java.util.concurrent.atomic.AtomicInteger;
import org.HdrHistogram.Histogram;
import org.HdrHistogram.HistogramLogWriter;
import org.HdrHistogram.Recorder;
import com.yahoo.ycsb.measurements.exporter.MeasurementsExporter;
/**
* Take measurements and maintain a HdrHistogram of a given metric, such as
* READ LATENCY.
*
* @author nitsanw
*
*/
public class OneMeasurementHdrHistogram extends OneMeasurement {
// we need one log per measurement histogram
final PrintStream log;
final HistogramLogWriter histogramLogWriter;
final Recorder histogram = new Recorder(3);
Histogram totalHistogram;
public OneMeasurementHdrHistogram(String name, Properties props) {
super(name);
boolean shouldLog = Boolean.parseBoolean(props.getProperty("hdrhistogram.fileoutput", "false"));
if (!shouldLog) {
log = null;
histogramLogWriter = null;
return;
}
try {
final String hdrOutputFilename = props.getProperty("hdrhistogram.output.path", "") +name+".hdr";
log = new PrintStream(new FileOutputStream(hdrOutputFilename), false);
} catch (FileNotFoundException e) {
throw new RuntimeException("Failed to open hdr histogram output file", e);
}
histogramLogWriter = new HistogramLogWriter(log);
histogramLogWriter.outputComment("[Logging for: " + name + "]");
histogramLogWriter.outputLogFormatVersion();
histogramLogWriter.outputStartTime(System.currentTimeMillis());
histogramLogWriter.outputLegend();
}
/**
* It appears latency is reported in micros.
* Using {@link Recorder} to support concurrent updates to histogram.
*
* @see com.yahoo.ycsb.OneMeasurement#measure(int)
*/
public void measure(int latencyInMicros) {
histogram.recordValue(latencyInMicros);
}
/**
* This is called from a main thread, on orderly termination.
*
* @see com.yahoo.ycsb.measurements.OneMeasurement#exportMeasurements(com.yahoo.ycsb.measurements.exporter.MeasurementsExporter)
*/
@Override
public void exportMeasurements(MeasurementsExporter exporter) throws IOException {
// accumulate the last interval which was not caught by status thread
Histogram intervalHistogram = getIntervalHistogramAndAccumulate();
if(histogramLogWriter != null) {
histogramLogWriter.outputIntervalHistogram(intervalHistogram);
// we can close now
log.close();
}
exporter.write(getName(), "Operations", totalHistogram.getTotalCount());
exporter.write(getName(), "AverageLatency(us)", totalHistogram.getMean());
exporter.write(getName(), "MinLatency(us)", totalHistogram.getMinValue());
exporter.write(getName(), "MaxLatency(us)", totalHistogram.getMaxValue());
exporter.write(getName(), "95thPercentileLatency(ms)", totalHistogram.getValueAtPercentile(90)/1000);
exporter.write(getName(), "99thPercentileLatency(ms)", totalHistogram.getValueAtPercentile(99)/1000);
for (Map.Entry<Integer, AtomicInteger> entry : returncodes.entrySet()) {
exporter.write(getName(), "Return=" + entry.getKey(), entry.getValue().get());
}
}
/**
* This is called periodically from the StatusThread. There's a single StatusThread per Client process.
* We optionally serialize the interval to log on this opportunity.
* @see com.yahoo.ycsb.measurements.OneMeasurement#getSummary()
*/
@Override
public String getSummary() {
Histogram intervalHistogram = getIntervalHistogramAndAccumulate();
// we use the summary interval as the histogram file interval.
if(histogramLogWriter != null) {
histogramLogWriter.outputIntervalHistogram(intervalHistogram);
}
DecimalFormat d = new DecimalFormat("#.##");
return "[" + getName() +
": Count=" + intervalHistogram.getTotalCount() +
", Max=" + intervalHistogram.getMaxValue() +
", Min=" + intervalHistogram.getMinValue() +
", Avg=" + d.format(intervalHistogram.getMean()) +
", 90=" + d.format(intervalHistogram.getValueAtPercentile(90)) +
", 99=" + d.format(intervalHistogram.getValueAtPercentile(99)) +
", 99.9=" + d.format(intervalHistogram.getValueAtPercentile(99.9)) +
", 99.99=" + d.format(intervalHistogram.getValueAtPercentile(99.99)) +"]";
}
private Histogram getIntervalHistogramAndAccumulate() {
Histogram intervalHistogram = histogram.getIntervalHistogram();
// add this to the total time histogram.
if (totalHistogram == null) {
totalHistogram = intervalHistogram;
}
else {
totalHistogram.add(intervalHistogram);
}
return intervalHistogram;
}
}
core/src/main/java/com/yahoo/ycsb/measurements/OneMeasurementHistogram.java
+ 145
− 145
View file @ a9bc84c4
/**
* Copyright (c) 2010 Yahoo! Inc. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you
* may not use this file except in compliance with the License. You
* may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
* implied. See the License for the specific language governing
* permissions and limitations under the License. See accompanying
* LICENSE file.
*/
package com.yahoo.ycsb.measurements;
import java.io.IOException;
import java.text.DecimalFormat;
import java.util.HashMap;
import java.util.Properties;
import com.yahoo.ycsb.measurements.exporter.MeasurementsExporter;
/**
* Take measurements and maintain a histogram of a given metric, such as READ LATENCY.
*
* @author cooperb
*
*/
public class OneMeasurementHistogram extends OneMeasurement
{
public static final String BUCKETS="histogram.buckets";
public static final String BUCKETS_DEFAULT="1000";
int _buckets;
int[] histogram;
int histogramoverflow;
int operations;
long totallatency;
//keep a windowed version of these stats for printing status
int windowoperations;
long windowtotallatency;
int min;
int max;
public OneMeasurementHistogram(String name, Properties props)
{
super(name);
_buckets=Integer.parseInt(props.getProperty(BUCKETS, BUCKETS_DEFAULT));
histogram=new int[_buckets];
histogramoverflow=0;
operations=0;
totallatency=0;
windowoperations=0;
windowtotallatency=0;
min=-1;
max=-1;
}
/* (non-Javadoc)
* @see com.yahoo.ycsb.OneMeasurement#measure(int)
*/
public synchronized void measure(int latency)
{
if (latency/1000>=_buckets)
{
histogramoverflow++;
}
else
{
histogram[latency/1000]++;
}
operations++;
totallatency+=latency;
windowoperations++;
windowtotallatency+=latency;
if ( (min<0) || (latency<min) )
{
min=latency;
}
if ( (max<0) || (latency>max) )
{
max=latency;
}
}
@Override
public void exportMeasurements(MeasurementsExporter exporter) throws IOException
{
exporter.write(getName(), "Operations", operations);
exporter.write(getName(), "AverageLatency(us)", (((double)totallatency)/((double)operations)));
exporter.write(getName(), "MinLatency(us)", min);
exporter.write(getName(), "MaxLatency(us)", max);
int opcounter=0;
boolean done95th=false;
for (int i=0; i<_buckets; i++)
{
opcounter+=histogram[i];
if ( (!done95th) && (((double)opcounter)/((double)operations)>=0.95) )
{
exporter.write(getName(), "95thPercentileLatency(ms)", i);
done95th=true;
}
if (((double)opcounter)/((double)operations)>=0.99)
{
exporter.write(getName(), "99thPercentileLatency(ms)", i);
break;
}
}
for (Map.Entry<Integer, AtomicInteger> entry : returncodes.entrySet()) {
exporter.write(getName(), "Return=" + entry.getKey(), entry.getValue().get());
}
for (int i=0; i<_buckets; i++)
{
exporter.write(getName(), Integer.toString(i), histogram[i]);
}
exporter.write(getName(), ">"+_buckets, histogramoverflow);
}
@Override
public String getSummary() {
if (windowoperations==0)
{
return "";
}
DecimalFormat d = new DecimalFormat("#.##");
double report=((double)windowtotallatency)/((double)windowoperations);
windowtotallatency=0;
windowoperations=0;
return "["+getName()+" AverageLatency(us)="+d.format(report)+"]";
}
}
/**
* Copyright (c) 2010 Yahoo! Inc. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you
* may not use this file except in compliance with the License. You
* may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
* implied. See the License for the specific language governing
* permissions and limitations under the License. See accompanying
* LICENSE file.
*/
package com.yahoo.ycsb.measurements;
import java.io.IOException;
import java.text.DecimalFormat;
import java.util.Properties;
import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger;
import com.yahoo.ycsb.measurements.exporter.MeasurementsExporter;
/**
* Take measurements and maintain a histogram of a given metric, such as READ LATENCY.
*
* @author cooperb
*
*/
public class OneMeasurementHistogram extends OneMeasurement
{
public static final String BUCKETS="histogram.buckets";
public static final String BUCKETS_DEFAULT="1000";
int _buckets;
int[] histogram;
int histogramoverflow;
int operations;
long totallatency;
//keep a windowed version of these stats for printing status
int windowoperations;
long windowtotallatency;
int min;
int max;
public OneMeasurementHistogram(String name, Properties props)
{
super(name);
_buckets=Integer.parseInt(props.getProperty(BUCKETS, BUCKETS_DEFAULT));
histogram=new int[_buckets];
histogramoverflow=0;
operations=0;
totallatency=0;
windowoperations=0;
windowtotallatency=0;
min=-1;
max=-1;
}
/* (non-Javadoc)
* @see com.yahoo.ycsb.OneMeasurement#measure(int)
*/
public synchronized void measure(int latency)
{
if (latency/1000>=_buckets)
{
histogramoverflow++;
}
else
{
histogram[latency/1000]++;
}
operations++;
totallatency+=latency;
windowoperations++;
windowtotallatency+=latency;
if ( (min<0) || (latency<min) )
{
min=latency;
}
if ( (max<0) || (latency>max) )
{
max=latency;
}
}
@Override
public void exportMeasurements(MeasurementsExporter exporter) throws IOException
{
exporter.write(getName(), "Operations", operations);
exporter.write(getName(), "AverageLatency(us)", (((double)totallatency)/((double)operations)));
exporter.write(getName(), "MinLatency(us)", min);
exporter.write(getName(), "MaxLatency(us)", max);
int opcounter=0;
boolean done95th=false;
for (int i=0; i<_buckets; i++)
{
opcounter+=histogram[i];
if ( (!done95th) && (((double)opcounter)/((double)operations)>=0.95) )
{
exporter.write(getName(), "95thPercentileLatency(ms)", i);
done95th=true;
}
if (((double)opcounter)/((double)operations)>=0.99)
{
exporter.write(getName(), "99thPercentileLatency(ms)", i);
break;
}
}
for (Map.Entry<Integer, AtomicInteger> entry : returncodes.entrySet()) {
exporter.write(getName(), "Return=" + entry.getKey(), entry.getValue().get());
}
for (int i=0; i<_buckets; i++)
{
exporter.write(getName(), Integer.toString(i), histogram[i]);
}
exporter.write(getName(), ">"+_buckets, histogramoverflow);
}
@Override
public String getSummary() {
if (windowoperations==0)
{
return "";
}
DecimalFormat d = new DecimalFormat("#.##");
double report=((double)windowtotallatency)/((double)windowoperations);
windowtotallatency=0;
windowoperations=0;
return "["+getName()+" AverageLatency(us)="+d.format(report)+"]";
}
}
core/src/main/java/com/yahoo/ycsb/measurements/OneMeasurementTimeSeries.java
+ 163
− 162
View file @ a9bc84c4
/**
* Copyright (c) 2010 Yahoo! Inc. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you
* may not use this file except in compliance with the License. You
* may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
* implied. See the License for the specific language governing
* permissions and limitations under the License. See accompanying
* LICENSE file.
*/
package com.yahoo.ycsb.measurements;
import java.io.IOException;
import java.text.DecimalFormat;
import java.util.HashMap;
import java.util.Properties;
import java.util.Vector;
import com.yahoo.ycsb.measurements.exporter.MeasurementsExporter;
class SeriesUnit
{
/**
* @param time
* @param average
*/
public SeriesUnit(long time, double average) {
this.time = time;
this.average = average;
}
public long time;
public double average;
}
/**
* A time series measurement of a metric, such as READ LATENCY.
*/
public class OneMeasurementTimeSeries extends OneMeasurement
{
/**
* Granularity for time series; measurements will be averaged in chunks of this granularity. Units are milliseconds.
*/
public static final String GRANULARITY="timeseries.granularity";
public static final String GRANULARITY_DEFAULT="1000";
int _granularity;
Vector<SeriesUnit> _measurements;
long start=-1;
long currentunit=-1;
int count=0;
int sum=0;
int operations=0;
long totallatency=0;
//keep a windowed version of these stats for printing status
int windowoperations=0;
long windowtotallatency=0;
int min=-1;
int max=-1;
public OneMeasurementTimeSeries(String name, Properties props)
{
super(name);
_granularity=Integer.parseInt(props.getProperty(GRANULARITY,GRANULARITY_DEFAULT));
_measurements=new Vector<SeriesUnit>();
}
void checkEndOfUnit(boolean forceend)
{
long now=System.currentTimeMillis();
if (start<0)
{
currentunit=0;
start=now;
}
long unit=((now-start)/_granularity)*_granularity;
if ( (unit>currentunit) || (forceend) )
{
double avg=((double)sum)/((double)count);
_measurements.add(new SeriesUnit(currentunit,avg));
currentunit=unit;
count=0;
sum=0;
}
}
@Override
public void measure(int latency)
{
checkEndOfUnit(false);
count++;
sum+=latency;
totallatency+=latency;
operations++;
windowoperations++;
windowtotallatency+=latency;
if (latency>max)
{
max=latency;
}
if ( (latency<min) || (min<0) )
{
min=latency;
}
}
@Override
public void exportMeasurements(MeasurementsExporter exporter) throws IOException
{
checkEndOfUnit(true);
exporter.write(getName(), "Operations", operations);
exporter.write(getName(), "AverageLatency(us)", (((double)totallatency)/((double)operations)));
exporter.write(getName(), "MinLatency(us)", min);
exporter.write(getName(), "MaxLatency(us)", max);
//TODO: 95th and 99th percentile latency
for (Map.Entry<Integer, AtomicInteger> entry : returncodes.entrySet()) {
exporter.write(getName(), "Return=" + entry.getKey(), entry.getValue().get());
}
for (SeriesUnit unit : _measurements)
{
exporter.write(getName(), Long.toString(unit.time), unit.average);
}
}
@Override
public String getSummary() {
if (windowoperations==0)
{
return "";
}
DecimalFormat d = new DecimalFormat("#.##");
double report=((double)windowtotallatency)/((double)windowoperations);
windowtotallatency=0;
windowoperations=0;
return "["+getName()+" AverageLatency(us)="+d.format(report)+"]";
}
}
/**
* Copyright (c) 2010 Yahoo! Inc. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you
* may not use this file except in compliance with the License. You
* may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
* implied. See the License for the specific language governing
* permissions and limitations under the License. See accompanying
* LICENSE file.
*/
package com.yahoo.ycsb.measurements;
import java.io.IOException;
import java.util.Vector;
import java.util.Properties;
import java.text.DecimalFormat;
import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger;
import com.yahoo.ycsb.measurements.exporter.MeasurementsExporter;
class SeriesUnit
{
/**
* @param time
* @param average
*/
public SeriesUnit(long time, double average) {
this.time = time;
this.average = average;
}
public long time;
public double average;
}
/**
* A time series measurement of a metric, such as READ LATENCY.
*/
public class OneMeasurementTimeSeries extends OneMeasurement
{
/**
* Granularity for time series; measurements will be averaged in chunks of this granularity. Units are milliseconds.
*/
public static final String GRANULARITY="timeseries.granularity";
public static final String GRANULARITY_DEFAULT="1000";
int _granularity;
Vector<SeriesUnit> _measurements;
long start=-1;
long currentunit=-1;
int count=0;
int sum=0;
int operations=0;
long totallatency=0;
//keep a windowed version of these stats for printing status
int windowoperations=0;
long windowtotallatency=0;
int min=-1;
int max=-1;
public OneMeasurementTimeSeries(String name, Properties props)
{
super(name);
_granularity=Integer.parseInt(props.getProperty(GRANULARITY,GRANULARITY_DEFAULT));
_measurements=new Vector<SeriesUnit>();
}
void checkEndOfUnit(boolean forceend)
{
long now=System.currentTimeMillis();
if (start<0)
{
currentunit=0;
start=now;
}
long unit=((now-start)/_granularity)*_granularity;
if ( (unit>currentunit) || (forceend) )
{
double avg=((double)sum)/((double)count);
_measurements.add(new SeriesUnit(currentunit,avg));
currentunit=unit;
count=0;
sum=0;
}
}
@Override
public void measure(int latency)
{
checkEndOfUnit(false);
count++;
sum+=latency;
totallatency+=latency;
operations++;
windowoperations++;
windowtotallatency+=latency;
if (latency>max)
{
max=latency;
}
if ( (latency<min) || (min<0) )
{
min=latency;
}
}
@Override
public void exportMeasurements(MeasurementsExporter exporter) throws IOException
{
checkEndOfUnit(true);
exporter.write(getName(), "Operations", operations);
exporter.write(getName(), "AverageLatency(us)", (((double)totallatency)/((double)operations)));
exporter.write(getName(), "MinLatency(us)", min);
exporter.write(getName(), "MaxLatency(us)", max);
//TODO: 95th and 99th percentile latency
for (Map.Entry<Integer, AtomicInteger> entry : returncodes.entrySet()) {
exporter.write(getName(), "Return=" + entry.getKey(), entry.getValue().get());
}
for (SeriesUnit unit : _measurements)
{
exporter.write(getName(), Long.toString(unit.time), unit.average);
}
}
@Override
public String getSummary() {
if (windowoperations==0)
{
return "";
}
DecimalFormat d = new DecimalFormat("#.##");
double report=((double)windowtotallatency)/((double)windowoperations);
windowtotallatency=0;
windowoperations=0;
return "["+getName()+" AverageLatency(us)="+d.format(report)+"]";
}
}
core/src/main/java/com/yahoo/ycsb/measurements/TwoInOneMeasurement.java
+ 60
− 58
View file @ a9bc84c4
/**
* Copyright (c) 2010 Yahoo! Inc. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you
* may not use this file except in compliance with the License. You
* may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
* implied. See the License for the specific language governing
* permissions and limitations under the License. See accompanying
* LICENSE file.
/**
* Copyright (c) 2010 Yahoo! Inc. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you
* may not use this file except in compliance with the License. You
* may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
* implied. See the License for the specific language governing
* permissions and limitations under the License. See accompanying
* LICENSE file.
*/
package com.yahoo.ycsb.measurements;
......@@ -29,51 +29,53 @@ import com.yahoo.ycsb.measurements.exporter.MeasurementsExporter;
*
*/
public class TwoInOneMeasurement extends OneMeasurement {
final OneMeasurement thing1,thing2;
public TwoInOneMeasurement(String name, OneMeasurement thing1,OneMeasurement thing2) {
super(name);
this.thing1 = thing1;
this.thing2 = thing2;
}
/**
* No need for synchronization, using CHM to deal with that
*
* @see com.yahoo.ycsb.OneMeasurement#reportReturnCode(int)
*/
public void reportReturnCode(int code) {
thing1.reportReturnCode(code);
}
final OneMeasurement thing1,thing2;
public TwoInOneMeasurement(String name, OneMeasurement thing1,OneMeasurement thing2) {
super(name);
this.thing1 = thing1;
this.thing2 = thing2;
}
/**
* No need for synchronization, using CHM to deal with that
*
* @see com.yahoo.ycsb.OneMeasurement#reportReturnCode(int)
*/
public void reportReturnCode(int code) {
thing1.reportReturnCode(code);
}
/**
* It appears latency is reported in micros.
* Using {@link Recorder} to support concurrent updates to histogram.
*
* @see com.yahoo.ycsb.OneMeasurement#measure(int)
*/
public void measure(int latencyInMicros) {
thing1.measure(latencyInMicros);
thing2.measure(latencyInMicros);
}
/**
* It appears latency is reported in micros.
* Using {@link Recorder} to support concurrent updates to histogram.
*
* @see com.yahoo.ycsb.OneMeasurement#measure(int)
*/
public void measure(int latencyInMicros) {
thing1.measure(latencyInMicros);
thing2.measure(latencyInMicros);
}
/**
* This is called from a main thread, on orderly termination.
*
* @see com.yahoo.ycsb.measurements.OneMeasurement#exportMeasurements(com.yahoo.ycsb.measurements.exporter.MeasurementsExporter)
*/
@Override
public void exportMeasurements(MeasurementsExporter exporter) throws IOException {
thing1.exportMeasurements(exporter);
thing2.exportMeasurements(exporter);
}
/**
* This is called from a main thread, on orderly termination.
*
* @see com.yahoo.ycsb.measurements.OneMeasurement#exportMeasurements(com.yahoo.ycsb.measurements.exporter.MeasurementsExporter)
*/
@Override
public void exportMeasurements(MeasurementsExporter exporter) throws IOException {
thing1.exportMeasurements(exporter);
thing2.exportMeasurements(exporter);
}
/**
* This is called periodically from the StatusThread. There's a single StatusThread per Client process.
* We optionally serialize the interval to log on this opportunity.
* @see com.yahoo.ycsb.measurements.OneMeasurement#getSummary()
*/
@Override
public String getSummary() {
return thing1.getSummary() + "\n" + thing2.getSummary();
}
/**
* This is called periodically from the StatusThread. There's a single StatusThread per Client process.
* We optionally serialize the interval to log on this opportunity.
* @see com.yahoo.ycsb.measurements.OneMeasurement#getSummary()
*/
@Override
public String getSummary() {
return thing1.getSummary() + "\n" + thing2.getSummary();
}
}
core/src/main/java/com/yahoo/ycsb/measurements/exporter/JSONArrayMeasurementsExporter.java
+ 0
− 1
View file @ a9bc84c4
......@@ -36,7 +36,6 @@ public class JSONArrayMeasurementsExporter implements MeasurementsExporter
public JSONArrayMeasurementsExporter(OutputStream os) throws IOException
{
BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(os));
g = factory.createJsonGenerator(bw);
g.setPrettyPrinter(new DefaultPrettyPrinter());
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment