A Minimal CSV writer for data collection

Problem

Write (a subset of) the data to a CSV file during data collection.

Approach

Write a callback function that integrates Python’s built-in csv module with bluesky.

Example Solution

Boiler plate imports and configuration

import path
import os
import bluesky as bs
import bluesky.plans as bp
import bluesky.callbacks as bc
import csv
from ophyd.sim import motor, det

import matplotlib.pyplot as plt


# Do this if running the example interactively;
# skip it when building the documentation.
import os
if 'BUILDING_DOCS' not in os.environ:
    from bluesky.utils import install_qt_kicker  # for notebooks, qt -> nb
    install_qt_kicker()
    plt.ion()
    det.exposure_time = .1  # simulate detector exposure time

RE = bs.RunEngine({})

Define a callback class which writes out a CSV file

class CSVWriter(bc.CallbackBase):
    def __init__(self, fields, fname_format, fpath):
        self._path = path.Path(fpath)
        os.makedirs(self._path, exist_ok=True)
        self._fname_fomat = fname_format
        self._fields = fields
        self._writer = None
        self._fout = None

    def close(self):
        if self._fout is not None:
            self._fout.close()
        self._fout = None
        self._writer = None

    def start(self, doc):
        self.close()

        fname = self._path / self._fname_fomat.format(**doc)

        self._fout = open(fname, 'xt')
        self._writer = csv.writer(self._fout)

    def descriptor(self, doc):
        if self._writer is not None:
            self._writer.writerow(self._fields)

    def event(self, doc):
        data = doc['data']
        if self._writer is not None:
            self._writer.writerow(data[k] for k in self._fields)

    def stop(self, doc):
        self.close()

Set up some callbacks

def create_cbs():
    return [bc.LiveTable([motor, det]), bc.LivePlot('det', 'motor')]

fmt = '{user}_{uid:.6s}.csv'
export_path = '/tmp/export_demo'
csv_writer = CSVWriter(('motor', 'det'), fmt, export_path)

# send all documents to the CSV writer
RE.subscribe('all', csv_writer)

Out:

/home/travis/virtualenv/python3.6.7/lib/python3.6/site-packages/bluesky/run_engine.py:2388: UserWarning: The order of the arguments has been changed. Because the meaning of the arguments is unambiguous, the old usage will continue to work indefinitely, but the new usage is encouraged: call subscribe(func, name) instead of subscribe(name, func). Additionally, the 'name' argument has become optional. Its default value is 'all'.
  warn("The order of the arguments has been changed. Because the "

run the scan

uid, = RE(bp.scan([det], motor, -5, 5, 11),
          create_cbs(), user='tcaswell')

Out:

+-----------+------------+------------+----------------+------------+
|   seq_num |       time |      motor | motor_setpoint |        det |
+-----------+------------+------------+----------------+------------+
|         1 | 21:00:53.2 |     -5.000 |         -5.000 |      0.000 |
|         2 | 21:00:53.2 |     -4.000 |         -4.000 |      0.000 |
|         3 | 21:00:53.3 |     -3.000 |         -3.000 |      0.011 |
|         4 | 21:00:53.3 |     -2.000 |         -2.000 |      0.135 |
|         5 | 21:00:53.4 |     -1.000 |         -1.000 |      0.607 |
|         6 | 21:00:53.4 |      0.000 |          0.000 |      1.000 |
|         7 | 21:00:53.4 |      1.000 |          1.000 |      0.607 |
|         8 | 21:00:53.5 |      2.000 |          2.000 |      0.135 |
|         9 | 21:00:53.5 |      3.000 |          3.000 |      0.011 |
|        10 | 21:00:53.5 |      4.000 |          4.000 |      0.000 |
|        11 | 21:00:53.6 |      5.000 |          5.000 |      0.000 |
+-----------+------------+------------+----------------+------------+
generator scan ['9ffb9459'] (scan num: 1)

check file

fname = os.path.join(export_path,
                     '{user}_{uid:.6s}.csv'.format(user='tcaswell', uid=uid))

print("--- {} ---".format(fname))
with open(fname, 'r') as fin:
    for ln in fin:
        print(ln.strip())

Out:

--- /tmp/export_demo/tcaswell_9ffb94.csv ---
motor,det
-5.0,3.726653172078671e-06
-4.0,0.00033546262790251185
-3.0,0.011108996538242306
-2.0,0.1353352832366127
-1.0,0.6065306597126334
0.0,1.0
1.0,0.6065306597126334
2.0,0.1353352832366127
3.0,0.011108996538242306
4.0,0.00033546262790251185
5.0,3.726653172078671e-06