File readout_fe.cpp
File List > midas_fe > readout_fe.cpp
Go to the documentation of this file
#include <stdio.h>
#include <stdlib.h>
#include <sys/mman.h>
#include <unistd.h>
#include <iomanip>
#include <iostream>
#include <list>
#include <sstream>
#include <string>
// clang-format off
#include "midas.h"
// clang-format on
#include <chrono>
#include "DummyFEBSlowcontrolInterface.h"
#include "FEBSlowcontrolInterface.h"
#include "mcstd.h"
#include "mfe.h"
#include "missing_hardware.h"
#include "msystem.h"
#include "mudaq_device.h"
#include "odb_setup.h"
#include "odbxx.h"
#include "utils.h"
// MIDAS settings
const char* frontend_name = "Readout";
const char* frontend_file_name = __FILE__;
BOOL equipment_common_overwrite = TRUE;
// Readout variables
volatile uint32_t* dma_buf;
uint32_t* dma_buf_local;
uint32_t reset_regs = 0;
uint16_t eventID_data = 301;
uint32_t readout_state_regs = 0;
bool use_software_dummy = false;
uint32_t n_mevents = 0;
uint32_t readout_timeout = 1000;
uint32_t use_timeout = true;
uint32_t cnt_loop = 0;
uint32_t maxwords = 0;
mudaq::DmaMudaqDevice* mup = nullptr;
mudaq::DmaMudaqDevice::DataBlock block;
std::vector<uint32_t> lvds_banks;
std::map<uint64_t, std::list<mevent_t>> mevents;
midas::odb m_settings;
bool saw_readout_enabled = false;
static void print_swb_counters(mudaq::DmaMudaqDevice& mu) {
// counter / rate
// 0-3: input link subheader cnt / rate
// 4-7: input link hit cnt / rate
// 8-11: input link package cnt / rate
// 12: mux word cnt / rate
printf("Input subheader (cnt / rate (Hz))\n");
for (int i = 0; i <= 3; ++i) {
mu.write_register(SWB_COUNTER_REGISTER_W, i);
uint32_t cnt = mu.read_register_ro(SWB_COUNTER_REGISTER_R);
uint32_t rate = mu.read_register_ro(SWB_LINK_COUNTER_REGISTER_R);
printf("Link:%i %i / %i\n", i, cnt, rate);
}
printf("Input hit (cnt / rate (Hz))\n");
for (int i = 4; i <= 7; ++i) {
mu.write_register(SWB_COUNTER_REGISTER_W, i);
uint32_t cnt = mu.read_register_ro(SWB_COUNTER_REGISTER_R);
uint32_t rate = mu.read_register_ro(SWB_LINK_COUNTER_REGISTER_R);
printf("Link:%i %i / %i\n", i, cnt, rate);
}
printf("Input package (cnt / rate (Hz))\n");
for (int i = 8; i <= 11; ++i) {
mu.write_register(SWB_COUNTER_REGISTER_W, i);
uint32_t cnt = mu.read_register_ro(SWB_COUNTER_REGISTER_R);
uint32_t rate = mu.read_register_ro(SWB_LINK_COUNTER_REGISTER_R);
printf("Link:%i %i / %i\n", i, cnt, rate);
}
mu.write_register(SWB_COUNTER_REGISTER_W, 12);
uint32_t cnt = mu.read_register_ro(SWB_COUNTER_REGISTER_R);
uint32_t rate = mu.read_register_ro(SWB_LINK_COUNTER_REGISTER_R);
printf("MUX out (cnt / rate (Hz)):%i / %i\n", cnt, rate);
printf("DMA hit cnt out: %i \n",
mu.read_register_ro(EVENT_BUILD_IDLE_NOT_HEADER_R) * 4); // hit cnt to DMA
printf("DMA hit rate out: %i \n",
mu.read_register_ro(EVENT_BUILD_TAG_FIFO_FULL_R)); // fifo rate to DMA
printf("DMA skip hit cnt: %i \n",
mu.read_register_ro(EVENT_BUILD_SKIP_EVENT_DMA_R) * 4); // hit drop DMA busy
printf("DMA FIFO full: %i \n", mu.read_register_ro(BUFFER_STATUS_REGISTER_R)); // fifo full cnt
}
uint64_t generate_random_pixel_hit_swb(bool print) {
uint8_t tot = rand() % 32; // 0 to 31
uint8_t chipID = rand() % 16;// 0 to 15
uint8_t col = rand() % 256; // 0 to 255
uint8_t row = rand() % 250; // 0 to 249
uint32_t time = rand();
uint64_t hit =
((uint64_t)(0 & 0x1) << 63) |
((uint64_t)(chipID & 0x3) << 61) |
(uint64_t)time;
return hit;
}
int init_mudaq(mudaq::MudaqDevice& mu) {
#ifdef NO_A10_BOARD
#else
int fd = open("/dev/mudaq0_dmabuf", O_RDWR);
if (fd < 0) {
printf("fd = %d\n", fd);
return FE_ERR_DRIVER;
}
dma_buf = reinterpret_cast<uint32_t*>(
mmap(nullptr, MUDAQ_DMABUF_DATA_LEN, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0));
#endif
if (dma_buf == MAP_FAILED) {
cm_msg1(MERROR, "quads", "frontend_init", "mmap failed: dmabuf = %p\n", MAP_FAILED);
return FE_ERR_DRIVER;
}
dma_buf_local = new (std::align_val_t(8)) uint32_t[MUDAQ_DMABUF_DATA_LEN];
// open mudaq
if (!mu.open()) {
std::cout << "Could not open device " << std::endl;
cm_msg1(MERROR, "quads", "frontend_init", "Could not open device");
return FE_ERR_DRIVER;
}
// check mudaq
if (!mu.is_ok())
return FE_ERR_DRIVER;
else {
cm_msg1(MINFO, "quads", "frontend_init", "Mudaq device is ok");
}
// switch off the data generator (just in case ..)
mu.write_register(DATAGENERATOR_REGISTER_W, 0x0);
usleep(2000);
// set DMA_CONTROL_W
mu.write_register(DMA_CONTROL_W, 0x0);
return SUCCESS;
}
int begin_of_run() {
// setup readout state register
readout_state_regs = 0;
// get copy of setting
m_settings.connect("/Equipment/Quads/Settings");
#ifdef NO_A10_BOARD
#else
mudaq::DmaMudaqDevice& mu = *mup;
// set all in reset
mu.write_register_wait(RESET_REGISTER_W, reset_regs, 100);
// empty dma buffer
for (uint32_t i = 0; i < dma_buf_nwords; i++) dma_buf[i] = 0;
#endif
#ifdef NO_A10_BOARD
#else
if ((bool)m_settings["Readout"]["Datagen Enable"]) {
// setup data generator
cm_msg1(MINFO, "quads", "readout_fe", "Use datagenerator with divider register %i",
(int)m_settings["Readout"]["Datagen Divider"]);
mu.write_register(DATAGENERATOR_DIVIDER_REGISTER_W,
(int)m_settings["Readout"]["Datagen Divider"]);
readout_state_regs = SET_USE_BIT_GEN_LINK(readout_state_regs);
}
#endif
if ((bool)m_settings["Readout"]["use_merger"]) {
// readout merger
cm_msg1(MINFO, "quads", "readout_fe", "Use Time Merger");
readout_state_regs = SET_USE_BIT_MERGER(readout_state_regs);
} else {
// readout stream
cm_msg1(MINFO, "quads", "readout_fe", "Use Stream Merger");
readout_state_regs = SET_USE_BIT_STREAM(readout_state_regs);
}
if ((bool)m_settings["Readout"]["use_send_time"]) {
cm_msg1(MINFO, "quads", "readout_fe", "Use send time as header time");
readout_state_regs = SET_USE_BIT_SEND_TIME(readout_state_regs);
}
readout_state_regs = SET_USE_BIT_GENERIC(readout_state_regs);
use_software_dummy = (bool)m_settings["Readout"]["Software dummy"];
n_mevents = (int)m_settings["Readout"]["n_mevents"];
#ifdef NO_A10_BOARD
#else
// write readout register
mu.write_register(SWB_READOUT_STATE_REGISTER_W, readout_state_regs);
// request to read blocks of 256 bits
maxwords = (uint32_t) m_settings["Readout"]["max_requested_words"];
mu.write_register(GET_N_DMA_WORDS_REGISTER_W,
(uint32_t) m_settings["Readout"]["max_requested_words"]);
// set event id for this frontend
mu.write_register(FARM_EVENT_ID_REGISTER_W, eventID_data);
// link masks
mu.write_register(SWB_GENERIC_MASK_REGISTER_W, (uint32_t) m_settings["Readout"]["mask_n_generic"]);
// release reset
mu.write_register_wait(RESET_REGISTER_W, 0x0, 100);
#endif
mevents.clear();
saw_readout_enabled = false;
return SUCCESS;
}
int end_of_run() { return SUCCESS; }
int frontend_exit_user() {
#ifdef NO_A10_BOARD
#else
if (mup) {
mup->disable();
mup->close();
delete mup;
}
#endif
return SUCCESS;
}
int create_midas_events(uint32_t* dmaBuffer, uint32_t dmaBufSize, int rbh)
{
if (!dmaBuffer)
return -1;
if (dmaBufSize < 2)
return -1;
// sort hits
static constexpr uint64_t kTimestampMask = (1ULL << 37) - 1ULL; // bits 0..36
struct HitWord {
uint32_t word0;
uint32_t word1;
uint64_t ts;
};
std::vector<HitWord> hits;
for (uint32_t i = 0; i < maxwords * 4; i += 2) {
uint32_t word0 = dmaBuffer[i];
uint32_t word1 = dmaBuffer[i+1];
uint64_t word =
static_cast<uint64_t>(word0) |
(static_cast<uint64_t>(word1) << 32);
uint64_t ts = word & kTimestampMask;
hits.push_back({word0, word1, ts});
}
// Sort by timestamp
std::sort(hits.begin(), hits.end(),
[](const HitWord& a, const HitWord& b) { return a.ts < b.ts; });
// create MIDAS event
void* event = nullptr;
int status = 0;
do {
if(!is_readout_thread_enabled()) return -1;
if(!readout_enabled()) {
cm_msg1(MERROR, "quads", "create_midas_events()", "we are not running");
return -1;
}
status = rb_get_wp(rbh, &event, 0);
if(status == DB_TIMEOUT) { ss_sleep(10); }
else if(status != DB_SUCCESS) return -1;
} while(status == DB_TIMEOUT);
if(!event) {
cm_msg1(MERROR, "quads", "create_midas_events", "unexpected nullptr from rb_get_wp\n");
return -1;
}
auto eventHeader = reinterpret_cast<EVENT_HEADER*>(event);
bm_compose_event_threadsafe(eventHeader, eventID_data, 0, 0, &equipment[0].serial_number);
auto bankHeader = reinterpret_cast<BANK_HEADER*>(eventHeader + 1);
bk_init32a(bankHeader); // create MIDAS bank
uint32_t* data = nullptr;
std::string bank_name = "H000";
bk_create(bankHeader, bank_name.c_str(), TID_UINT32, reinterpret_cast<void**>(&data));
// memcpy(data, dmaBuffer, dmaBufSize * sizeof(uint32_t));
// data += dmaBufSize * sizeof(uint32_t);
// Store 64-bit words as two uint32_t words
for (size_t i = 0; i < hits.size(); ++i) {
data[2 * i] = hits[i].word0;
data[2 * i + 1] = hits[i].word1;
}
memcpy(data, data, hits.size() / 2 * sizeof(uint32_t));
data += hits.size() / 2 * sizeof(uint32_t);
bk_close(bankHeader, data);
eventHeader->data_size = bk_size(bankHeader);
rb_increment_wp(rbh, sizeof(EVENT_HEADER) + eventHeader->data_size);
return SUCCESS;
}
int read_stream_thread(void*) {
// get mudaq
mudaq::DmaMudaqDevice& mu = *mup;
// tell framework that we are alive
signal_readout_thread_active(0, TRUE);
// obtain ring buffer for inter-thread data exchange
int rbh = get_event_rbh(0);
int status;
// timeout for DMA
bool timeout = false;
// dummy buffer for test data
int nHits = 5000;
std::vector<uint32_t> dma_buf_dummy32;
std::vector<uint64_t> dma_buf_dummy64;
std::chrono::steady_clock::time_point begin = std::chrono::steady_clock::now();
// actuall readout loop
while (is_readout_thread_enabled()) {
if (!timeout)
begin = std::chrono::steady_clock::now();
// don't readout events if we are not running
if (!readout_enabled()) {
// printf("Not running!\n");
// do not produce events when run is stopped
ss_sleep(10); // don't eat all CPU
continue;
}
// we generate the events in software
if (use_software_dummy) {
// create dummy hits
uint64_t first_hit = generate_random_pixel_hit_swb(true);
dma_buf_dummy64.push_back(first_hit);
for (int i = 0; i < nHits; i++)
dma_buf_dummy64.push_back(generate_random_pixel_hit_swb(false));
// Convert 64-bit words to two 32-bit words each
dma_buf_dummy32.reserve(dma_buf_dummy64.size() * 2);
for (uint64_t word : dma_buf_dummy64) {
dma_buf_dummy32.push_back(static_cast<uint32_t>(word & 0xFFFFFFFF)); // lower 32 bits
dma_buf_dummy32.push_back(static_cast<uint32_t>((word >> 32) & 0xFFFFFFFF)); // upper 32 bits
}
// printf("hit64:hit32: %llx %x %x\n", dma_buf_dummy64[0], dma_buf_dummy32[0], dma_buf_dummy32.data()[0]);
// create MIDAS events
create_midas_events(dma_buf_dummy32.data(), dma_buf_dummy32.size(), rbh);
dma_buf_dummy64.clear();
dma_buf_dummy32.clear();
ss_sleep(300); // limit data rate
continue;
}
// start dma
if (!timeout)
mu.enable_continous_readout(0);
// wait for requested data
cnt_loop = 0;
timeout = false;
while ((mu.read_register_ro(EVENT_BUILD_STATUS_REGISTER_R) & 1) == 0) {
if (use_timeout && cnt_loop++ >= readout_timeout) {
timeout = true;
break;
}
if (!readout_enabled())
break; // TODO: we break here hard later the firmware should stop at run end
ss_sleep(10);
}
// dont read from the buffer if the status is not done
if (timeout)
continue;
// disable dma
mu.disable();
// get written words from FPGA in bytes
uint32_t size_dma_buf = mu.last_endofevent_addr() * 256 / 8;
uint32_t maxidx = (mu.last_endofevent_addr() + 1) * 8 - 1;
uint32_t last_written = mu.last_written_addr();
std::chrono::steady_clock::time_point end = std::chrono::steady_clock::now();
double dma_time = std::chrono::duration_cast<std::chrono::microseconds>(end - begin).count();
std::cout << "Time difference (DMA) = "
<< std::chrono::duration_cast<std::chrono::microseconds>(end - begin).count()
<< "[µs]" << std::endl;
begin = std::chrono::steady_clock::now();
if (size_dma_buf > MUDAQ_DMABUF_DATA_LEN) {
cm_msg1(MERROR, "quads", "ro_swb_fe", "Read invalid DMA buffer size %i!\n", size_dma_buf);
continue;
}
// [AK] NOTE: use direct copy as memcpy does not arantee
// non-optimization for volatile
// [MK] NOTE: max words is in 256bit words
for (uint32_t i = 0; i < maxwords * 8; i++) {
dma_buf_local[i] = dma_buf[i];
}
// create MIDAS events
create_midas_events(dma_buf_local, mu.last_endofevent_addr(), rbh);
end = std::chrono::steady_clock::now();
double event_time = (double) std::chrono::duration_cast<std::chrono::microseconds>(end - begin).count();
std::cout << "Time difference (EVENT) = "
<< std::chrono::duration_cast<std::chrono::microseconds>(end - begin).count()
<< "[µs]" << std::endl;
m_settings["Readout"]["HitRate"] = (double) maxwords * 4 / (dma_time / 1e6);
}
return SUCCESS;
}
int frontend_init() {
// get copy of setting
m_settings.connect("/Equipment/Quads/Settings");
// setup max event size
set_max_event_size(dma_buf_size);
// end and start of run
install_begin_of_run(begin_of_run);
install_end_of_run(end_of_run);
install_frontend_exit(frontend_exit_user);
// init dma and mudaq device
mup = new mudaq::DmaMudaqDevice("/dev/mudaq0");
int status = init_mudaq(*mup);
if (status != SUCCESS)
return FE_ERR_DRIVER;
// switch off and reset DMA for now
mup->disable();
// set reset registers
reset_regs = SET_RESET_BIT_DATA_PATH(reset_regs);
reset_regs = SET_RESET_BIT_DATAGEN(reset_regs);
reset_regs = SET_RESET_BIT_SWB_TIME_MERGER(reset_regs);
reset_regs = SET_RESET_BIT_SWB_STREAM_MERGER(reset_regs);
// create ring buffer for readout thread
create_event_rb(0);
// create readout thread
ss_thread_create(read_stream_thread, NULL);
// Set our transition sequence. The default is 500.
cm_set_transition_sequence(TR_START, 300);
// Set our transition sequence. The default is 500. Setting it
// to 700 means we are called AFTER most other clients.
cm_set_transition_sequence(TR_STOP, 700);
// set write cache to 10MB
// set_cache_size("SYSTEM", 10000000);
return SUCCESS;
}
EQUIPMENT equipment[] = {{
"Readout", /* equipment name */
{eventID_data, 0, /* event ID, trigger mask */
"SYSTEM", /* event buffer */
EQ_USER, /* equipment type */
0, /* event source */
"MIDAS", /* format */
TRUE, /* enabled */
RO_RUNNING, /* read always, except during
transistions and update ODB */
1000, /* read every 1 sec */
0, /* stop run after this event limit */
0, /* number of sub events */
0, /* log history every event */
"", "", ""},
NULL, /* readout routine */
},
{""}};