commit 5c3473ff1fe5629496d84e4a1fdfba610b094e13 Author: Dominic Höglinger Date: Mon Apr 28 22:09:19 2025 +0200 Initial release of Streaming Event Trace diff --git a/LICENSE.md b/LICENSE.md new file mode 100644 index 0000000..666bb1b --- /dev/null +++ b/LICENSE.md @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2025 Dominic Hoeglinger + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..75ea4ac --- /dev/null +++ b/README.md @@ -0,0 +1,100 @@ +``` + ▓▓▓▓▓ ░░░ ███████ ███████ ████████ + ▓ ▓ ▒ ░ ██ ██ ██ Streaming Event Trace +▓ ▓ ▓ ░░ ███████ █████ ██ A tiny signal tracing library + ▓ ▓ ▒ ░ ██ ██ ██ + ▓▓▓▓▓ ░░░ ███████ ███████ ██ Version 1.0.1 Alpha +``` + +# Introduction + +SET is a small tracing library written in C99. +It is highly optimized for minimal data usage and aims to be easy to use +while also being flexible enough to be integrated in hard real time systems. + +# How to use + +SET requires the integrator to implement a few functions such as the timestamp and output functions in order to work. +Apart from that, it is up to the specific use case how to stream data. +In its simplest form, traces are processed with no real time demands on a dedicated interface. +For more advanced usage, this library allows packets to be written, as the time budget allows, +into any given buffer to send its contents via DMA to for instance a serial output. + +Simple example on Arduino: + +````c + +#include "set.h" + +// Basic implementation of output and timestamp retrieval +void set_out(const char* const str, size_t len) { Serial.write(str, len); } +uint32_t set_timestamp(void) { return micros(); } + +// No concurrency protection needed +void set_crit_on(uint32_t *h) {} +void set_crit_off(const uint32_t h) {} + +static const size_t s_tracesize = 256; +static set_trace_t s_tracebuf[256]; + +void setup() +{ + set_init(s_tracebuf, s_tracesize); + set_enable(set_drop); // drop traces on full buffer + + set_u8trace("Serial.Ready", 0, false); + Serial.begin(921600); + while (!Serial); + set_u8trace("Serial.Ready", 1, false); +} + +void loop() +{ + unsigned sawtooth = 0; + + if (sawtooth < 0xFFFF) sawtooth++; + else sawtooth = 0; + set_u32trace("Main.Sawtooth", sawtooth, false); + + if (sawtooth == 4711) Serial.write("I can still be used with regular text."); + + set_output(0, true); //send unlimited traces compressed + set_diagtrace(); // add diagnostic traces +} +```` + +# Basic operating principle + +Traces are identified by a signal name, which may be hierarchical such as `ExampleModule.Function.Value`. +These must be constant C strings as their pointers are stored in the trace buffer. +Once a trace is rendered, the signal name is hashed using the BSD sum function. +This allows a client to identify a trace packet according to the source tree scanned. + +For sending the traces to a client, they are rendered into a packet. +A packet consist of a preamble, a flag byte, frame data and an epilouge. +The preamble and epilouge are chosen to be the ANSI escape sequences for storing the current position, and restoring it. + +The frame data may be compressed using fastlz, which is indicated by the Z flag in the packet. +Regardless of compression, it is COBS encoded with delimiter 0033 to not interfere with the preamble or epilouge. + +Frame data consists of multiple frames, COBS encoded with delimiter 0 and delimited with the same value. +Each frame contains the signal hash, its type and depending on it a value ranging from 1-4 bytes and indexing information. +Timestamps are sent in increments to conserve bandwidth. +All trace functions feature a parameter which enables timestamp capture, if it is omitted this means no time delta is sent, +making the trace appear to coincide with the last trace which captured a timestamp. + +# The recorder utility + +The repository contains an example client application which captures packages and outputs the traces in a Value Change Dump. +This script uses the PyVCD package which is the only dependency. +Apart from that, this utility has every function integrated it'll need to decode a trace stream. +It expects data via stdin, and displays any non-tracing traffic. + +Example usage tracing remotely via SSH: +````bash +# Open /dev/ttyUSB0 via SSH and record to "trace.vcd" with timescale of 1 microsecond with diagnostics enabled +ssh user@host -C "picocom -b 921600 /dev/ttyUSB0" | ./set_record.py -d trace.vcd -s ./Sources -t "1 us" --diagnostics +```` + +The resulting VCD can be inspected with dedicated programs such as GTKwave. + diff --git a/set.c b/set.c new file mode 100644 index 0000000..a0781ca --- /dev/null +++ b/set.c @@ -0,0 +1,872 @@ +/* + * PET: Pocket Event Trace - A tiny tracing library + * Copyright (C) 2025 Dominic Hoeglinger + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include "pet.h" + +#define SET_NIBBLE_TO_HEX(nibble) ((nibble) < 10 ? (nibble) + '0' : (nibble) - 10 + 'A') +#define SET_NIBBLE(value, nibble) ((value >> nibble) & 0xF) +#define SET_BYTE(value, offset) (char)(((value >> (8 * offset)) & 0xFF)) +#define SET_ENCODE(value, nibble) (char)(SET_NIBBLE_TO_HEX(SET_NIBBLE(value, nibble))) +#define SET_NELEMS(a) (sizeof(a) / sizeof(a[0])) +#define SET_MACROPACK_FRAME_SIZE (2 * 8) +#define SET_MACROPACK_SIZE(n) (SET_MACROPACK_FRAME_SIZE * n + (SET_MACROPACK_FRAME_SIZE * n) / 254 + 1 + n - 1) +#define SET_MAX_RENDER_SIZE(n) (SET_MACROPACK_SIZE(n) + (SET_MACROPACK_SIZE(n) / 254) + 1) + +#define FASTLZ1_MAX_COPY 32 +#define FASTLZ1_MAX_LEN 264 /* 256 + 8 */ +#define FASTLZ1_MAX_L1_DISTANCE 8192 +#define FASTLZ1_MAX_L2_DISTANCE 8191 +#define FASTLZ1_MAX_FARDISTANCE (65535 + FASTLZ1_MAX_L2_DISTANCE - 1) + +#define FASTLZ1_HASH_LOG 13 +#define FASTLZ1_HASH_SIZE (1 << FASTLZ1_HASH_LOG) +#define FASTLZ1_HASH_MASK (FASTLZ1_HASH_SIZE - 1) + +typedef enum +{ + set_d1 = 0x11, + set_d2 = 0x12, + set_d4 = 0x14, + set_v1 = 0x21, + set_v2 = 0x22, + set_v4 = 0x24, + set_a1 = 0x31, + set_a2 = 0x32, + set_a4 = 0x34, + set_s4 = 0x44, + set_f4 = 0x54, + set_ev = 0x60, +} set_type_t; + +typedef enum +{ + set_packet_z = (1 << 0), +} set_packetflags_t; + +typedef struct +{ + const char* m_tag; + uint16_t m_hash; +} set_hashcache_t; + +typedef union +{ + uint32_t u32value; +#if (!SET_FLOAT_DISABLE) + float f32value; +#endif +} set_convert_t; + +static bool s_enabled = false; +static uint32_t s_last_time = 0; +set_overflow_policy_t s_policy = set_overwrite; +static set_trace_t* s_tracebuffer; +static size_t s_tracebuffer_size = 0; +static size_t s_tracebuffer_head = 0; +static size_t s_tracebuffer_tail = 0; +static size_t s_tracebuffer_items = 0; +static bool s_tracebuffer_full = false; +static bool s_locked = false; +static bool s_tty_enabled = false; +static size_t s_tty_rubout_len = 0; + +static uint32_t s_diag_avg_compression_total = 0; +static uint32_t s_diag_avg_compression_n = 0; +static uint32_t s_diag_items_sent = 0; + +static uint32_t s_diag_avg_ctime_total = 0; +static uint32_t s_diag_avg_ctime_n = 0; + +static uint32_t s_diag_avg_rtime_total = 0; +static uint32_t s_diag_avg_rtime_n = 0; + +#if (SET_HASHCACHE_LINES > 0) +static set_hashcache_t s_hashcache[SET_HASHCACHE_LINES] = {0}; +static size_t s_hashcache_index = 0; +#endif + +size_t render_macropacket_payload(char buffer[], const size_t buffer_size, size_t n); +static size_t frame_preamble(char buffer[], const size_t buffer_size); +static size_t frame_epilouge(char buffer[], const size_t buffer_size); +static size_t pack_frame(char packet_bytes[], const set_trace_t* const p_packet); +static size_t ttyize_payload(char packet_buffer[], const size_t packet_size, const size_t buffer_size); +static uint16_t bsd_hash(const char* const str, size_t n); +static bool populate_time_delta_packet(const uint32_t timestamp, set_trace_t* const p_trace); +static void tracebuffer_add(const uint32_t timestamp, const set_type_t type, const uint32_t value, const uint8_t sub, char const *tag); +size_t cobs_encode(uint8_t* dst, size_t dst_buf_len, const uint8_t* ptr, int len, uint8_t delim); +static int fastlz1_compress(const void* input, int length, void* output); + +void set_init(set_trace_t tracebuffer[], size_t ntraces) +{ + s_tracebuffer = tracebuffer; + s_tracebuffer_size = ntraces; +} + +size_t set_get_buffer_items(void) +{ + return s_tracebuffer_items; +} + +void set_tty_mode(const bool enable) +{ + s_tty_enabled = enable; +} + +size_t set_tty_rubout(char buffer[], const size_t buffer_size) +{ + const size_t rubout_packet_size = 3 + s_tty_rubout_len + 3; + size_t buffer_pos = 0; + if (buffer_size < rubout_packet_size) + { + return 0; + } + buffer_pos += frame_preamble(buffer, buffer_size); + for (size_t i = 0; i < s_tty_rubout_len; ++i) + { + buffer[buffer_pos + i] = ' '; + } + buffer_pos += s_tty_rubout_len; + buffer_pos += frame_epilouge(&buffer[buffer_pos], buffer_size - buffer_pos); + + s_tty_rubout_len = 0; + + return buffer_pos; +} + +size_t set_output(size_t n, const bool compress) +{ + const size_t render_number_items = 4; + const size_t render_max_size = SET_MAX_RENDER_SIZE(render_number_items); + char packet_bytes[SET_MAX_RENDER_SIZE(4)] = {0}; + size_t packet_size = 0; + size_t i = 0; + n = (n == 0) ? s_tracebuffer_size : n; + + while ((s_tracebuffer_full || (s_tracebuffer_tail != s_tracebuffer_head)) && (i < n)) + { + packet_size = set_render(packet_bytes, render_max_size, render_number_items, compress); + i += render_number_items; + + set_out(packet_bytes, packet_size); + } + + packet_size = set_tty_rubout(packet_bytes, render_max_size); + set_out(packet_bytes, packet_size); + + return i; +} + +size_t render_macropacket_payload(char buffer[], const size_t buffer_size, size_t n) +{ + const size_t max_render_size = (8 + 1 + 1) * 2 - 1; + char payload_buffer[8] = {0}; + char delimimter = 0x00; + set_trace_t timestamp_pack = {0}; + size_t payload_size = 0; + size_t i = 0; + size_t buffer_pos = 0; + size_t encode_size = 0; + n = (n == 0) ? s_tracebuffer_size : n; + + while ((s_tracebuffer_full || (s_tracebuffer_tail != s_tracebuffer_head)) && (i < n) && ((buffer_pos + max_render_size) < buffer_size)) + { + payload_size = pack_frame(payload_buffer, &s_tracebuffer[s_tracebuffer_tail]); + encode_size = cobs_encode(&buffer[buffer_pos], buffer_size - buffer_pos, payload_buffer, payload_size, delimimter); + buffer_pos += encode_size; + buffer[buffer_pos] = delimimter; + buffer_pos++; + + if (populate_time_delta_packet(s_tracebuffer[s_tracebuffer_tail].m_timestamp, ×tamp_pack)) + { + + payload_size = pack_frame(payload_buffer, ×tamp_pack); + encode_size = cobs_encode(&buffer[buffer_pos], buffer_size - buffer_pos, payload_buffer, payload_size, delimimter); + + buffer_pos += encode_size; + buffer[buffer_pos] = delimimter; + buffer_pos++; + } + + s_tracebuffer_tail = (s_tracebuffer_tail + 1) % s_tracebuffer_size; + s_tracebuffer_full = false; + s_tracebuffer_items = (s_tracebuffer_items > 0) ? (s_tracebuffer_items - 1) : 0; + i++; + s_diag_items_sent++; + } + + // remove last delimiter + if(buffer_pos) buffer_pos--; + + return buffer_pos; +} + +size_t set_render(char buffer[], const size_t buffer_size, const size_t n, const bool compress) +{ + const char delimimter = '\033'; + size_t n_elems = (n == 0) ? s_tracebuffer_items : n; + n_elems = (s_tty_enabled && (n > 4)) ? 4 : n_elems; + size_t render_max_size = SET_MAX_RENDER_SIZE(n_elems); + uint32_t ticks_render_start = set_timestamp(); + char *final_payload_buffer = NULL; + size_t final_payload_size = 0; + + while ((buffer_size < (1 + (render_max_size * 2))) && (n_elems > 0)) + { + n_elems--; + render_max_size = SET_MAX_RENDER_SIZE(n_elems); + } + + if (n_elems == 0) return 0; + + if (compress) + { + size_t macropack_payload_size = render_macropacket_payload(buffer, render_max_size, n_elems); + if (macropack_payload_size == 0) return 0; + + size_t compressed_size = macropack_payload_size + (macropack_payload_size*15)/100; + char *compressed_buffer = &buffer[buffer_size - compressed_size]; + uint32_t ticks_comp_start = set_timestamp(); + compressed_size = fastlz1_compress(buffer, macropack_payload_size, compressed_buffer); + final_payload_buffer = compressed_buffer; + final_payload_size = compressed_size; + s_diag_avg_ctime_total += set_timestamp() - ticks_comp_start; + s_diag_avg_ctime_n++; + + if (macropack_payload_size > 0) + { + s_diag_avg_compression_total += 100 - ((100 * compressed_size) / macropack_payload_size); + s_diag_avg_compression_n++; + } + } + else + { + char *macropack_buffer = &buffer[buffer_size - render_max_size]; + size_t macropack_payload_size = render_macropacket_payload(macropack_buffer, render_max_size, n_elems); + if (macropack_payload_size == 0) return 0; + + final_payload_buffer = macropack_buffer; + final_payload_size = macropack_payload_size; + } + + + size_t out_pos = frame_preamble(buffer, buffer_size); + + uint8_t flags = 0; + if (compress) flags |= (uint8_t)set_packet_z; + buffer[out_pos] = flags; + out_pos++; + + size_t payload_size = cobs_encode(&buffer[out_pos], buffer_size - out_pos, final_payload_buffer, final_payload_size, delimimter); + + if (s_tty_enabled == true) + { + payload_size = ttyize_payload(&buffer[out_pos], payload_size, buffer_size - out_pos); + if (payload_size > s_tty_rubout_len) + { + s_tty_rubout_len = payload_size; + } + } + + out_pos += payload_size; + out_pos += frame_epilouge(&buffer[out_pos], buffer_size - out_pos); + + s_diag_avg_rtime_total += set_timestamp() - ticks_render_start; + s_diag_avg_rtime_n++; + + return out_pos; +} + +void set_enable(const set_overflow_policy_t policy) +{ + s_policy = policy; + s_enabled = true; +} + +void set_disable(void) +{ + s_enabled = false; +} + +void set_clear(void) +{ + s_tracebuffer_tail = s_tracebuffer_head; + s_tracebuffer_full = false; +} + +void set_diagtrace(void) +{ + const uint32_t buffer_health = ((s_tracebuffer_size - s_tracebuffer_items) * 0xFF)/s_tracebuffer_size; + + set_u32trace("PET.BufferItems", s_tracebuffer_items, false); + set_u8trace("PET.BufferHealth", (uint8_t)buffer_health, true); + + if (s_diag_avg_compression_n > 0) + { + const uint32_t average = s_diag_avg_compression_total / s_diag_avg_compression_n; + set_u8trace("PET.CompressionLevel", (uint8_t)average, true); + s_diag_avg_compression_total = 0; + s_diag_avg_compression_n = 0; + } + + if (s_diag_items_sent > 0) + { + set_u32trace("PET.ItemsSent", s_diag_items_sent, true); + s_diag_items_sent = 0; + } + + if (s_diag_avg_ctime_n > 0) + { + const uint32_t average = s_diag_avg_ctime_total / s_diag_avg_ctime_n; + set_u8trace("PET.CompressionTime", (uint8_t)average, true); + s_diag_avg_ctime_total = 0; + s_diag_avg_ctime_n = 0; + } + + + if (s_diag_avg_rtime_n > 0) + { + const uint32_t average = s_diag_avg_rtime_total / s_diag_avg_rtime_n; + set_u8trace("PET.RenderTime", (uint8_t)average, true); + s_diag_avg_rtime_total = 0; + s_diag_avg_rtime_n = 0; + } +} + +void set_evtrace(const char* const tag, const bool skip_time) +{ + if (s_enabled == true) + { + tracebuffer_add(skip_time ? 0UL : set_timestamp(), set_ev, 1UL, 0U, tag); + } +} + +void set_u8trace(const char* const tag, const uint8_t value, const bool skip_time) +{ + if (s_enabled == true) + { + tracebuffer_add(skip_time ? 0UL : set_timestamp(), set_v1, (uint32_t)value, 0U, tag); + } +} + +void set_u16trace(const char* const tag, const uint16_t value, const bool skip_time) +{ + if (s_enabled == true) + { + tracebuffer_add(skip_time ? 0UL : set_timestamp(), set_v2, (uint32_t)value, 0U, tag); + } +} + +void set_u32trace(const char* const tag, const uint32_t value, const bool skip_time) +{ + if (s_enabled == true) + { + tracebuffer_add(skip_time ? 0UL : set_timestamp(), set_v4, (uint32_t)value, 0U, tag); + } +} + +void set_s8trace(const char* const tag, const int8_t value, const bool skip_time) +{ + if (s_enabled == true) + { + tracebuffer_add(skip_time ? 0UL : set_timestamp(), set_v1, (uint32_t)value, 0U, tag); + } +} + +void set_s16trace(const char* const tag, const int16_t value, const bool skip_time) +{ + if (s_enabled == true) + { + tracebuffer_add(skip_time ? 0UL : set_timestamp(), set_v2, (uint32_t)value, 0U, tag); + } +} + +void set_s4trace(const char* const tag, const int32_t value, const bool skip_time) +{ + if (s_enabled == true) + { + tracebuffer_add(skip_time ? 0UL : set_timestamp(), set_v4, (uint32_t)value, 0U, tag); + } +} + +void set_au8trace(const char* const tag, const uint8_t value[const], const uint8_t size, const bool skip_time) +{ + if (s_enabled == true) + { + const size_t ts = skip_time ? 0UL : set_timestamp(); + for (size_t i = 0; i < size; ++i) + { + tracebuffer_add(ts, set_a1, (uint32_t)value[i], (uint8_t)i, tag); + } + } +} + +void set_au16trace(const char* const tag, const uint16_t value[const], const uint8_t size, const bool skip_time) +{ + if (s_enabled == true) + { + const size_t ts = skip_time ? 0UL : set_timestamp(); + for (size_t i = 0; i < size; ++i) + { + tracebuffer_add(ts, set_a2, (uint32_t)value[i], (uint8_t)i, tag); + } + } +} + +void set_au32trace(const char* const tag, const uint32_t value[const], const uint8_t size, const bool skip_time) +{ + if (s_enabled == true) + { + const size_t ts = skip_time ? 0UL : set_timestamp(); + for (size_t i = 0; i < size; ++i) + { + uint16_t sub = (i) | (size << 8); + tracebuffer_add(ts, set_a4, (uint32_t)value[i], (uint8_t)i, tag); + } + } +} + +void set_as8trace(const char* const tag, const int8_t value[const], const uint8_t size, const bool skip_time) +{ + if (s_enabled == true) + { + const size_t ts = skip_time ? 0UL : set_timestamp(); + for (size_t i = 0; i < size; ++i) + { + tracebuffer_add(ts, set_a1, (uint32_t)value[i], (uint8_t)i, tag); + } + } +} + +void set_as16trace(const char* const tag, const int16_t value[const], const uint8_t size, const bool skip_time) +{ + if (s_enabled == true) + { + const size_t ts = skip_time ? 0UL : set_timestamp(); + for (size_t i = 0; i < size; ++i) + { + tracebuffer_add(ts, set_a2, (uint32_t)value[i], (uint8_t)i, tag); + } + } +} + +void set_as32trace(const char* const tag, const int32_t value[const], const uint8_t size, const bool skip_time) +{ + if (s_enabled == true) + { + const size_t ts = skip_time ? 0UL : set_timestamp(); + for (size_t i = 0; i < size; ++i) + { + uint16_t sub = (i) | (size << 8); + tracebuffer_add(ts, set_a4, (uint32_t)value[i], (uint8_t)i, tag); + } + } +} + +void set_strtrace(const char* const tag, const char* const str, const bool skip_time) +{ + if (s_enabled == true) + { + const size_t ts = skip_time ? 0UL : set_timestamp(); + uint32_t packword = 0; + uint8_t packindex = 0; + for (size_t i = 0; str[i] != '\0'; ++i) + { + packindex = (i % 4); + packword |= (str[i] << (8 * packindex)); + if (packindex == 3) + { + tracebuffer_add(ts, set_s4, packword, (uint8_t)str[i + 1] == '\0', tag); + packword = 0; + } + } + if (packindex != 3) + { + tracebuffer_add(ts, set_s4, packword, (uint8_t)true, tag); + } + } +} + +#if (!SET_FLOAT_DISABLE) +void set_f32trace(const char* const tag, const float value, const bool skip_time) +{ + set_convert_t converter = {.f32value = value }; + if (s_enabled == true) + { + tracebuffer_add(skip_time ? 0UL : set_timestamp(), set_f4, converter.u32value, 0U, tag); + } +} +#endif + +static size_t frame_preamble(char buffer[], const size_t buffer_size) +{ + size_t pack_pos = 0; + if (buffer_size >= 3) + { + buffer[pack_pos++] = '\033'; + buffer[pack_pos++] = '['; + buffer[pack_pos++] = 's'; + } + return pack_pos; +} + +static size_t frame_epilouge(char buffer[], const size_t buffer_size) +{ + size_t pack_pos = 0; + if (buffer_size >= 3) + { + buffer[pack_pos++] = '\033'; + buffer[pack_pos++] = '['; + buffer[pack_pos++] = 'u'; + } + return pack_pos; +} + +static size_t pack_frame(char packet_bytes[], const set_trace_t* const p_packet) +{ + size_t packet_size = 0; + + packet_bytes[packet_size++] = (char)p_packet->m_type; + switch ((set_type_t)p_packet->m_type) + { + case set_v1: + case set_a1: + case set_d1: + packet_bytes[packet_size++] = SET_BYTE(p_packet->m_value, 0); + break; + case set_v2: + case set_a2: + case set_d2: + packet_bytes[packet_size++] = SET_BYTE(p_packet->m_value, 0); + packet_bytes[packet_size++] = SET_BYTE(p_packet->m_value, 1); + break; + case set_v4: + case set_a4: + case set_s4: + case set_f4: + case set_d4: + packet_bytes[packet_size++] = SET_BYTE(p_packet->m_value, 0); + packet_bytes[packet_size++] = SET_BYTE(p_packet->m_value, 1); + packet_bytes[packet_size++] = SET_BYTE(p_packet->m_value, 2); + packet_bytes[packet_size++] = SET_BYTE(p_packet->m_value, 3); + break; + } + + switch ((set_type_t)p_packet->m_type) + { + case set_a1: + case set_a2: + case set_a4: + case set_s4: + packet_bytes[packet_size++] = (char)p_packet->m_sub; + break; + } + + switch ((set_type_t)p_packet->m_type) + { + case set_d1: + case set_d2: + case set_d4: + break; + default: + const uint16_t hash_tag = bsd_hash(p_packet->m_tag, SET_MAX_TAG_LEN); + packet_bytes[packet_size++] = SET_BYTE(hash_tag, 0); + packet_bytes[packet_size++] = SET_BYTE(hash_tag, 1); + break; + } + + return packet_size; +} + + +static size_t ttyize_payload(char packet_buffer[], const size_t packet_size, const size_t buffer_size) +{ + const size_t tty_packet_size = (packet_size * 2); + size_t backpos = 0; + char value = 0; + + if (tty_packet_size > buffer_size) return 0; + + for (size_t i = 0; i < packet_size; ++i) + { + value = packet_buffer[(packet_size - i - 1)]; + backpos = (packet_size - i - 1) * 2; + packet_buffer[backpos] = SET_ENCODE(value, 0); + packet_buffer[backpos + 1] = SET_ENCODE(value, 4); + } + + return tty_packet_size; +} + +static uint16_t bsd_hash(const char* const str, size_t n) +{ + uint32_t checksum = 0; + + if (str == NULL) return 0U; +#if (SET_HASHCACHE_LINES > 0) + for (size_t i = 0; i < SET_HASHCACHE_LINES; ++i) + { + size_t hash_index = (i + s_hashcache_index) % SET_HASHCACHE_LINES; + if (s_hashcache[hash_index].m_tag == str) + { + return s_hashcache[hash_index].m_hash; + } + } +#endif + for (size_t i = 0; (i < n) && (str[i] != '\0'); ++i) + { + checksum = (checksum >> 1) + ((checksum & 1) << 15); + checksum += str[i]; + checksum &= 0xFFFF; + } +#if (SET_HASHCACHE_LINES > 0) + s_hashcache_index = (s_hashcache_index + 1) % SET_HASHCACHE_LINES; + s_hashcache[s_hashcache_index].m_hash = checksum; + s_hashcache[s_hashcache_index].m_tag = str; +#endif + + return checksum; +} + +static bool populate_time_delta_packet(const uint32_t timestamp, set_trace_t* const p_trace) +{ + const uint32_t dt = timestamp - s_last_time; + uint16_t type = 0; + + if (timestamp == 0UL) return false; + + if (dt != 0) + { + if (dt > 0xFFFF) + { + type = set_d4; + } + else if (dt > 0xFF) + { + type = set_d2; + } + else + { + type = set_d1; + } + + p_trace->m_type = type; + p_trace->m_value = dt; + p_trace->m_sub = 0U; + p_trace->m_tag = NULL; + p_trace->m_timestamp = 0UL; + + s_last_time = timestamp; + return true; + } + + return false; +} + +static void tracebuffer_add(const uint32_t timestamp, const set_type_t type, const uint32_t value, const uint8_t sub, char const *tag) +{ + const size_t insertion_pos = s_tracebuffer_head; + uint32_t critical_h = 0UL; + + if (s_tracebuffer_full == false || s_policy == set_overwrite) + { + set_crit_on(&critical_h); + s_tracebuffer_head = (s_tracebuffer_head + 1) % s_tracebuffer_size; + s_tracebuffer_full = (s_tracebuffer_head == s_tracebuffer_tail); + s_tracebuffer_items++; + set_crit_off(critical_h); + + s_tracebuffer[insertion_pos].m_type = (uint8_t)type; + s_tracebuffer[insertion_pos].m_value = value; + s_tracebuffer[insertion_pos].m_sub = sub; + s_tracebuffer[insertion_pos].m_tag = tag; + s_tracebuffer[insertion_pos].m_timestamp = timestamp; + } +} + +size_t cobs_encode(uint8_t* dst, size_t dst_buf_len, const uint8_t* ptr, int len, uint8_t delim) +{ + size_t encode_pos = 1; + size_t ins = 0; + uint8_t code = 1; + bool add_last_code = true; + size_t dst_size = 0; + + for (size_t i = 0; i < len; ++i) + { + uint8_t byte = ptr[i]; + if (byte != 0) + { + dst[encode_pos] = byte ^ delim; + encode_pos++; + code = code + 1; + } + add_last_code = true; + if ((byte == 0) || (code == 0xFF)) + { + if (code == 0xFF) add_last_code = false; + dst[ins] = code ^ delim; + code = 1; + ins = encode_pos; + dst[encode_pos] = 0xFF ^ delim; + encode_pos++; + } + } + + if (add_last_code) + { + dst[ins] = code ^ delim; + dst[encode_pos] = delim; + encode_pos++; + } + else + { + dst[ins] = delim; + } + + encode_pos--; + return encode_pos; +} + +static uint32_t flz_readu32(const void* ptr) { + const uint8_t* p = (const uint8_t*)ptr; + return (p[3] << 24) | (p[2] << 16) | (p[1] << 8) | p[0]; +} + +static uint16_t flz_hash(uint32_t v) { + uint32_t h = (v * 2654435769LL) >> (32 - FASTLZ1_HASH_LOG); + return h & FASTLZ1_HASH_MASK; +} + + +static void set_memcpy(uint8_t* dest, const uint8_t* src, uint32_t count) +{ + for(size_t i = 0; i < count; ++i) + { + dest[i] = src[i]; + } +} + +static uint32_t flz_cmp(const uint8_t* p, const uint8_t* q, const uint8_t* r) { + const uint8_t* start = p; + while (q < r) + if (*p++ != *q++) break; + return p - start; +} + +static uint8_t* flz_literals(uint32_t runs, const uint8_t* src, uint8_t* dest) { + while (runs >= FASTLZ1_MAX_COPY) { + *dest++ = FASTLZ1_MAX_COPY - 1; + set_memcpy(dest, src, FASTLZ1_MAX_COPY); + src += FASTLZ1_MAX_COPY; + dest += FASTLZ1_MAX_COPY; + runs -= FASTLZ1_MAX_COPY; + } + if (runs > 0) { + *dest++ = runs - 1; + set_memcpy(dest, src, runs); + dest += runs; + } + return dest; +} + +static uint8_t* flz1_match(uint32_t len, uint32_t distance, uint8_t* op) { + --distance; + if (len > FASTLZ1_MAX_LEN - 2) + while (len > FASTLZ1_MAX_LEN - 2) { + *op++ = (7 << 5) + (distance >> 8); + *op++ = FASTLZ1_MAX_LEN - 2 - 7 - 2; + *op++ = (distance & 255); + len -= FASTLZ1_MAX_LEN - 2; + } + if (len < 7) { + *op++ = (len << 5) + (distance >> 8); + *op++ = (distance & 255); + } else { + *op++ = (7 << 5) + (distance >> 8); + *op++ = len - 7; + *op++ = (distance & 255); + } + return op; +} + +static int fastlz1_compress(const void* input, int length, void* output) { + const uint8_t* ip = (const uint8_t*)input; + const uint8_t* ip_start = ip; + const uint8_t* ip_bound = ip + length - 4; /* because readU32 */ + const uint8_t* ip_limit = ip + length - 12 - 1; + uint8_t* op = (uint8_t*)output; + + uint32_t htab[FASTLZ1_HASH_SIZE]; + uint32_t seq, hash; + + /* initializes hash table */ + for (hash = 0; hash < FASTLZ1_HASH_SIZE; ++hash) htab[hash] = 0; + + /* we start with literal copy */ + const uint8_t* anchor = ip; + ip += 2; + + /* main loop */ + while (ip < ip_limit) { + const uint8_t* ref; + uint32_t distance, cmp; + + /* find potential match */ + do { + seq = flz_readu32(ip) & 0xffffff; + hash = flz_hash(seq); + ref = ip_start + htab[hash]; + htab[hash] = ip - ip_start; + distance = ip - ref; + cmp = (distance < FASTLZ1_MAX_L1_DISTANCE) ? flz_readu32(ref) & 0xffffff : 0x1000000; + if (ip >= ip_limit) break; + ++ip; + } while (seq != cmp); + + if (ip >= ip_limit) break; + --ip; + + if (ip > anchor) { + op = flz_literals(ip - anchor, anchor, op); + } + + uint32_t len = flz_cmp(ref + 3, ip + 3, ip_bound); + op = flz1_match(len, distance, op); + + /* update the hash at match boundary */ + ip += len; + seq = flz_readu32(ip); + hash = flz_hash(seq & 0xffffff); + htab[hash] = ip++ - ip_start; + seq >>= 8; + hash = flz_hash(seq); + htab[hash] = ip++ - ip_start; + + anchor = ip; + } + + uint32_t copy = (uint8_t*)input + length - anchor; + op = flz_literals(copy, anchor, op); + + return op - (uint8_t*)output; +} diff --git a/set.h b/set.h new file mode 100644 index 0000000..1bcb458 --- /dev/null +++ b/set.h @@ -0,0 +1,285 @@ +/* + * PET: Pocket Event Trace - A tiny tracing library + * Copyright (C) 2025 Dominic Hoeglinger + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef SET_H_INCLUDED_ +#define SET_H_INCLUDED_ + +#include +#include +#include + +/** Maxmum signal name / tag length considered for the identifying hash */ +#define SET_MAX_TAG_LEN (126) + +/** Number of cached hashes */ +#define SET_HASHCACHE_LINES (64) + +/** Switch to enable or disable floating point support */ +#define SET_FLOAT_DISABLE (false) + +/** Overflow policy for the tracebuffer */ +typedef enum +{ + set_overwrite, /**< Overwrite oldest samples */ + set_drop /**< Drop newest samples */ +} set_overflow_policy_t; + +/** Element used to allocate the tracebuffer */ +typedef struct +{ + char const * m_tag; + uint32_t m_timestamp; + uint32_t m_value; + uint8_t m_type; + uint8_t m_sub; +} set_trace_t; + +/** @brief Integration hook writing out a packet to the desired interface + * @param[in] str Packet data + * @param[in] len Packet data length + * @return None + */ +void set_out(const char* const str, size_t len); + +/** @brief Integration hook for retrieving a timestamp + * @return Timestamp value + */ +uint32_t set_timestamp(void); + +/** @brief Integration hook which locks the tracebuffer + * This can be for instance a mutex or interrupt lock. + * @param[out] h handle value which will be passed to the #set_crit_off function + */ +void set_crit_on(uint32_t *h); + +/** @brief Integration hook which locks the tracebuffer + * This can be for instance a mutex or interrupt lock. + * @param[in] h handle value set by #set_crit_on + */ +void set_crit_off(const uint32_t h); + +/** @brief Initializes PET with the provided trace buffer + * @param[in] str Trace buffer array + * @param[in] len Trace buffer array length + * @return None + */ +void set_init(set_trace_t tracebuffer[], size_t ntraces); + +/** @brief Returns the current trace buffer utilization + * @return Number of items used in the trace buffer + */ +size_t set_get_buffer_items(void); + +/** @brief Enables or disables TTY mode + * TTY mode esures packet output is friendly for interactive console usage without a client. + * This means that at most 4 packets are sent at once, the data is encoded as hex digits + * and position is restored with each transmit. To erase a transmitted packet, see set_tty_rubout. + * @param[in] enable If true, enable TTY mode + * @return None + */ +void set_tty_mode(const bool enable); + +/** @brief Creates a rubout packet clearing the maximum amount of data transmitted in TTY mode. + * This consists of a ANSI store, enough spaces to overwrite sent data and an ANSI restore. + * @param[out] buffer output buffer to render the rubout packet + * @param[in] buffer_size the size of the output buffer + * @return The packet size written to the buffer + */ +size_t set_tty_rubout(char buffer[], const size_t buffer_size); + +/** @brief Enables tracing with the provided overflow policy + * @param[in] policy Either overwrite to overwrite oldest trace, or drop to drop the newest sample + * @return None + */ +void set_enable(const set_overflow_policy_t policy); + +/** @brief Disables tracing + * @return None + */ +void set_disable(void); + +/** @brief Clears the trace buffer deleting all captures samples + * @return None + */ +void set_clear(void); + +/** @brief Render a set amount of tracebuffer items and write them out using set_out + * This function is for basic usage where the output function writes into a managed stream such as stdout + * @param[in] n Number of tracebuffer items to render, or 0 for the current size of the tracebuffer + * @param[in] compress if true, render the packets compressed + * @return None + */ +size_t set_output(size_t n, const bool compress); + +/** @brief Render a set amount of tracebuffer items and store them in the provided buffer + * This advanced function is intended to be used with DMA buffers. + * @param[out] buffer output buffer to render the packets + * @param[in] buffer_size the size of the output buffer + * @param[in] n Number of tracebuffer items to render, or 0 for the current size of the tracebuffer + * @param[in] compress if true, render the packets compressed + * @return None + */ +size_t set_render(char buffer[], const size_t buffer_size, const size_t n, const bool compress); + +/** @brief Adds diagnostic samples to the tracebuffer + * Signals include: + * - PET.BufferItems:u32 - The current amount of tracebuffer items + * - PET.BufferHealth:u8 - The buffer utilization, zero means that the buffer is full while 255 means the buffer is free + * - PET.CompressionLevel:u8 - The average compression level if packets are rendered compressed + * - PET.CompressionTime:u32 - The average compression time spent for one packet render + * - PET.RenderTime:u32 - The average render time for one packet + * - PET.ItemsSent:u32 - The number of items sent since the last call to this function + * @return None + */ +void set_diagtrace(void); + +/** @brief Trace an event + * @param[in] tag Signal name of the trace + * @param[in] skip_time Use last recorded timestamp for this trace + * @return None + */ +void set_evtrace(const char* const tag, const bool skip_time); + +/** @brief Trace an u8 value + * @param[in] tag Signal name of the trace + * @param[in] value Value to trace + * @param[in] skip_time Use last recorded timestamp for this trace + * @return None + */ +void set_u8trace(const char* const tag, const uint8_t value, const bool skip_time); + +/** @brief Trace an u16 value + * @param[in] tag Signal name of the trace + * @param[in] value Value to trace + * @param[in] skip_time Use last recorded timestamp for this trace + * @return None + */ +void set_u16trace(const char* const tag, const uint16_t value, const bool skip_time); + +/** @brief Trace an u32 value + * @param[in] tag Signal name of the trace + * @param[in] value Value to trace + * @param[in] skip_time Use last recorded timestamp for this trace + * @return None + */ +void set_u32trace(const char* const tag, const uint32_t value, const bool skip_time); + +/** @brief Trace an s8 value + * @param[in] tag Signal name of the trace + * @param[in] value Value to trace + * @param[in] skip_time Use last recorded timestamp for this trace + * @return None + */ +void set_s8trace(const char* const tag, const int8_t value, const bool skip_time); + +/** @brief Trace an s16 value + * @param[in] tag Signal name of the trace + * @param[in] value Value to trace + * @param[in] skip_time Use last recorded timestamp for this trace + * @return None + */ +void set_s16trace(const char* const tag, const int16_t value, const bool skip_time); + +/** @brief Trace an s32 value + * @param[in] tag Signal name of the trace + * @param[in] value Value to trace + * @param[in] skip_time Use last recorded timestamp for this trace + * @return None + */ +void set_s32trace(const char* const tag, const int32_t value, const bool skip_time); + +/** @brief Trace an array of u8 values + * @param[in] tag Signal name of the trace + * @param[in] value Value array to trace + * @param[in] size Size of the value array, must be an integer literal for the client to detect the signal correctly + * @param[in] skip_time Use last recorded timestamp for this trace + * @return None + */ +void set_au8trace(const char* const tag, const uint8_t value[const], const uint8_t size, const bool skip_time); + +/** @brief Trace an array of u16 values + * @param[in] tag Signal name of the trace + * @param[in] value Value array to trace + * @param[in] size Size of the value array, must be an integer literal for the client to detect the signal correctly + * @param[in] skip_time Use last recorded timestamp for this trace + * @return None + */ +void set_au16trace(const char* const tag, const uint16_t value[const], const uint8_t size, const bool skip_time); + +/** @brief Trace an array of u32 values + * @param[in] tag Signal name of the trace + * @param[in] value Value array to trace + * @param[in] size Size of the value array, must be an integer literal for the client to detect the signal correctly + * @param[in] skip_time Use last recorded timestamp for this trace + * @return None + */ +void set_au32trace(const char* const tag, const uint32_t value[const], const uint8_t size, const bool skip_time); + + +/** @brief Trace an array of s8 values + * @param[in] tag Signal name of the trace + * @param[in] value Value array to trace + * @param[in] size Size of the value array, must be an integer literal for the client to detect the signal correctly + * @param[in] skip_time Use last recorded timestamp for this trace + * @return None + */ +void set_as8trace(const char* const tag, const int8_t value[const], const uint8_t size, const bool skip_time); + +/** @brief Trace an array of s16 values + * @param[in] tag Signal name of the trace + * @param[in] value Value array to trace + * @param[in] size Size of the value array, must be an integer literal for the client to detect the signal correctly + * @param[in] skip_time Use last recorded timestamp for this trace + * @return None + */ +void set_as16trace(const char* const tag, const int16_t value[const], const uint8_t size, const bool skip_time); + +/** @brief Trace an array of s32 values + * @param[in] tag Signal name of the trace + * @param[in] value Value array to trace + * @param[in] size Size of the value array, must be an integer literal for the client to detect the signal correctly + * @param[in] skip_time Use last recorded timestamp for this trace + * @return None + */ +void set_as32trace(const char* const tag, const int32_t value[const], const uint8_t size, const bool skip_time); + +/** @brief Trace a string + * @param[in] tag Signal name of the trace + * @param[in] str String to trace + * @param[in] skip_time Use last recorded timestamp for this trace + * @return None + */ +void set_strtrace(const char* const tag, const char* const str, const bool skip_time); + +#if (!SET_FLOAT_DISABLE) + +/** @brief Trace an f32 value + * @param[in] tag Signal name of the trace + * @param[in] value Value to trace + * @param[in] skip_time Use last recorded timestamp for this trace + * @return None + */ +void set_f32trace(const char* const tag, const float value, const bool skip_time); +#endif + +#endif diff --git a/set_record.py b/set_record.py new file mode 100644 index 0000000..66773df --- /dev/null +++ b/set_record.py @@ -0,0 +1,543 @@ +# -*- coding: utf-8 -*- + +import sys +import os +import re +import struct +import argparse +import datetime +from functools import partial +from vcd import VCDWriter, writer +from vcd.common import VarType + +header = """ + ▓▓▓▓▓ ░░░ ███████ ███████ ████████ + ▓ ▓ ▒ ░ ██ ██ ██ Streaming Event Trace +▓ ▓ ▓ ░░ ███████ █████ ██ VCD Recorder Utility + ▓ ▓ ▒ ░ ██ ██ ██ + ▓▓▓▓▓ ░░░ ███████ ███████ ██ (c) 2025 D.Hoeglinger + +""" + +def human_readable_size(size, decimal_places=2): + for unit in ['B', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB']: + if size < 1024.0 or unit == 'PiB': + break + size /= 1024.0 + return f"{size:.{decimal_places}f} {unit}" + +def is_hex(s): + return all(chr(c) in '0123456789abcdefABCDEF' for c in s) + +def bsd_hash(data): + checksum = 0 + for c in data: + checksum = (checksum >> 1) + ((checksum & 1) << 15) + checksum += ord(c) + checksum = checksum & 0xFFFF + return checksum + +def hex_to_int(hex_digit): + if hex_digit.isdigit(): + return int(hex_digit) + elif hex_digit in 'ABCDEF': + return ord(hex_digit) - ord('A') + 10 + elif hex_digit in 'abcdef': + return ord(hex_digit) - ord('a') + 10 + else: + raise ValueError("Invalid hexadecimal digit") + +def decode_hexstr(payload): + value = bytearray() + for i in range(0, len(payload), 2): + b = hex_to_int(chr(payload[i])) + b |= hex_to_int(chr(payload[i + 1])) << 4 + value.append(b) + return bytes(value) + +def hexstr(payload): + return payload.hex().upper() + +def decode_binstr(payload): + value = 0 + for i,b in enumerate(payload): + value |= b << (8 * i) + return value + +class DecodeError(Exception): + pass + +def cobs_decode(enc, delim): + enc = list(enc) + enc.append(delim) + enc2 = enc[0:-1] + length = len(enc2) + if delim != 0x00: + for i in range(0, len(enc2)): + enc2[i] = enc[i] ^ delim + dec = [] + code = 0xFF + block = 0 + for i in range(0, length): + byte = enc2[i] + if block != 0: + dec.append(byte) + else: + if (i + byte) > len(enc): + raise DecodeError(f"Marker pointing to end of packet, at {i}, marker={byte}") + if code != 0xFF: + dec.append(0) + code = byte + block = code + if code == 0: + break + block = block - 1 + return bytes(dec) + +def _memmove(data: bytearray, stidx: int, offset: int, mlen: int) -> None: + for i in range(mlen): + data[stidx + i] = data[stidx - offset + i] + +def fastlz_decompress_lv1(datain, doutlen): + opcode_0 = datain[0] + datain_idx = 1 + + dataout = bytearray(doutlen) + dataout_idx = 0; + + while True: + op_type = opcode_0 >> 5 + op_data = opcode_0 & 31 + + if op_type == 0b000: + # literal run + run = 1 + opcode_0 + dataout[dataout_idx:dataout_idx + run] = datain[datain_idx:datain_idx + run] + datain_idx += run + dataout_idx += run + + elif op_type == 0b111: + # long match + opcode_1 = datain[datain_idx] + datain_idx += 1 + opcode_2 = datain[datain_idx] + datain_idx += 1 + + match_len = 9 + opcode_1 + ofs = (op_data << 8) + opcode_2 + 1 + + _memmove(dataout, dataout_idx, ofs, match_len) + dataout_idx += match_len + + else: + # short match + opcode_1 = datain[datain_idx] + datain_idx += 1 + + match_len = 2 + op_type + ofs = (op_data << 8) + opcode_1 + 1 + + _memmove(dataout, dataout_idx, ofs, match_len) + dataout_idx += match_len + + if datain_idx < len(datain): + opcode_0 = datain[datain_idx] + datain_idx += 1 + else: + break + + return bytes(dataout[:dataout_idx]) + +def scan_for_signals(directory, predefined_signals): + library_files = ['pet.c', 'pet.h'] + rx_events = re.compile(r'pet_evtrace\(\"([^\"]+)\"') + rx_scalars = re.compile(r'pet_([usf])(8|16|32)trace\(\"([^\"]+)\"') + rx_arrays = re.compile(r'pet_a([us])(8|16|32)trace\(\"([^\"]+)\"\,\s*[^,]+,\s*((?:0x)?[a-zA-Z0-9]+)') + rx_strings = re.compile(r'pet_strtrace\(\"([^\"]+)\"') + signals = {} + valid = True + + def add_variable(tag, variable): + hash = bsd_hash(tag) + if hash in signals: + if signals[hash] != variable: + print(f"{file_path}: error: variable `{variable}` has conflicting hash with `{signals[hash]}`") + valid = False + else: + signals[hash] = variable + + for predef_signal in predefined_signals: + sig,_ = predef_signal.split(":") + add_variable(sig, predef_signal) + + for root, _, files in os.walk(directory): + for file in files: + if file.endswith(('.c', '.h')) and (file not in library_files): + file_path = os.path.join(root, file) + with open(file_path, 'r') as f: + content = f.read() + for match in rx_events.finditer(content): + tag = match.group(1) + variable = f"{tag}:event" + add_variable(tag, variable) + + for match in rx_scalars.finditer(content): + tag = match.group(3) + variable = f"{tag}:{match.group(1)}{match.group(2)}" + add_variable(tag, variable) + + for match in rx_arrays.finditer(content): + tag = match.group(3) + variable = f"{tag}:{match.group(1)}{match.group(2)}[{int(match.group(4),0)}]" + add_variable(tag, variable) + + for match in rx_strings.finditer(content): + tag = match.group(1) + variable = f"{tag}:string" + add_variable(tag, variable) + + return list(signals.values()), valid + +class Filter: + PREAMBLE = b"\x1b[s" + EPILOUGE = b"\x1b[u" + TAGCODE_LUT = { + 0x11 : "D1", + 0x12 : "D2", + 0x14 : "D4", + 0x21 : "V1", + 0x22 : "V2", + 0x24 : "V4", + 0x31 : "A1", + 0x32 : "A2", + 0x34 : "A4", + 0x44 : "S4", + 0x54 : "F4", + 0x60 : "EV" + } + + FLAGS_COMPRESSED = (1 << 0) + + def __init__(self, on_value, on_noise): + self.preamble_i = 0 + self.epilouge_i = 0 + self.packet_buffer = [] + self.noise_buffer = [] + self.process_value = on_value + self.process_noise = on_noise + self.packet_counter = 0 + self.packets_dropped = 0 + + def process(self, b): + if self.preamble_i == (len(self.PREAMBLE)): + self.packet_buffer.append(b) + if b == self.EPILOUGE[self.epilouge_i]: + self.epilouge_i += 1 + else: + self.epilouge_i = 0 + + if self.epilouge_i == (len(self.EPILOUGE)): + self.process_packet(self.packet_buffer[:-len(self.EPILOUGE)]) + self.packet_buffer = [] + self.preamble_i = 0 + self.epilouge_i = 0 + else: + if b == self.PREAMBLE[self.preamble_i]: + self.preamble_i += 1 + self.noise_buffer.append(b) + else: + self.preamble_i = 0 + for nb in self.noise_buffer: + self.process_noise(nb) + self.noise_buffer = [] + self.process_noise(b) + + def disassemble_packet(self, packet): + try: + tagcode = packet[0] + # ignore rubout packets + if chr(tagcode) == ' ': + return + + if tagcode not in self.TAGCODE_LUT: + self.packets_dropped += 1 + print("LUT ERR", tagcode) + return + + tag = self.TAGCODE_LUT[tagcode] + + value = None + offset = 1 + match tag: + case "D1"|"V1"|"A1": + value = decode_binstr(packet[offset:offset+1]) + offset += 1 + case "D2"|"V2"|"A2": + value = decode_binstr(packet[offset:offset+2]) + offset += 2 + case "D4"|"V4"|"A4"|"F4"|"S4": + value = decode_binstr(packet[offset:offset+4]) + offset += 4 + sub = None + if tag[0] == 'A' or tag[0] == 'S': + sub = decode_binstr(packet[offset:offset+1]) + offset += 1 + + hashtag = None + if tag[0] != 'D': + hashtag = decode_binstr(packet[-2:]) + except Exception as ex: + self.packets_dropped += 1 + return + + self.process_value(hashtag, value, sub, tag) + self.packet_counter += 1 + + def disassemble_macropacket(self, packet): + flags = packet[0] + payload = packet[1:] + + try: + macropack = cobs_decode(payload, 0x1B) + if self.FLAGS_COMPRESSED & flags: + frames = fastlz_decompress_lv1(macropack, 1024) + else: + frames = macropack + macropack_delim = 0x00 + for pack in frames.split(bytes(bytearray([macropack_delim]))): + if len(pack) == 0: + continue + try: + decoded = cobs_decode(pack, macropack_delim) + self.disassemble_packet(decoded) + except: + self.packets_dropped += 1 + except: + self.packets_dropped += 1 + + + def process_packet(self, packet): + if len(packet) == 0: + return + if is_hex(packet): + packet = decode_hexstr(packet) + if True or (len(packet) > 3 + 8 + 3): + self.disassemble_macropacket(packet) + else: + self.disassemble_packet(packet) + +class Retagger: + def __init__(self, on_value, tags=None): + tags = tags or [] + self._tag_lut = {bsd_hash(tag):tag for tag in tags} + self.process_value = on_value + self.packets_dropped = 0 + + def process(self, hashtag, number, sub, datatag): + if hashtag in self._tag_lut or datatag[0] == 'D': + tag = self._tag_lut.get(hashtag, None) + self.process_value(tag, number, sub, datatag) + else: + self.packets_dropped += 1 + +class VcdSink: + def __init__(self, fs, signals, timescale='1 us'): + self.writer = VCDWriter(fs, timescale=timescale, date=datetime.datetime.now().isoformat(), version=f"PET v1.0") + self.skalars = {} + self.arrays = {} + self.strings = {} + self.varnames = {} + self.timestamp = 0 + self.packets_dropped = 0 + for v in signals: + hvar, vtype = v.split(":") + hier, _, name = hvar.rpartition(".") + arr = None + s = vtype.split("[") + if len(s) == 2: + vtype, arr = s + dsize = 32 + dtype = 'integer' + match vtype: + case 'event': + dtype = 'event' + dsize = 32 + case 'f32': + dtype = 'real' + dsize = 32 + case 'u32'|'s32': + dtype = 'integer' + dsize = 32 + case 'u16'|'s16': + dtype = 'integer' + dsize = 16 + case 'u8'|'s8': + dtype = 'integer' + dsize = 8 + case 'string': + dtype = 'string' + dsize = 8 + + self.varnames[hvar] = hvar + + if arr is not None: + elems = int(arr.rstrip("]")) + vars = [] + for i in range(0, elems): + vars.append(self.writer.register_var(hvar, f"{name}[{i}:{(i+1)}]", 'wire', size=dsize)) + self.arrays[hvar] = vars + elif dtype == 'string': + self.strings[hvar] = [self.writer.register_var(hier, name, dtype, size=dsize), ""] + else: + self.skalars[hvar] = self.writer.register_var(hier, name, dtype, size=dsize) + + def process(self, tag, value, sub, datatag): + if datatag[0] == 'D': + self.timestamp += value + # array values + elif datatag[0] == 'A': + timestamp = self.timestamp + try: + #print(f"### {timestamp:012X} : {self.varnames[tag]}[{sub}] <= {value} [OK] ", flush=True) + self.writer.change(self.arrays[tag][sub], timestamp, value) + except ValueError: + print(f"### {timestamp:012X} : {self.varnames[tag]}[{sub}] <= {value} [VAL_ERR] ", flush=True) + self.packets_dropped += 1 + except writer.VCDPhaseError: + print(f"### {timestamp:012X} : {self.varnames[tag]}[{sub}] <= {value} [PHA_ERR] ", flush=True) + self.packets_dropped += 1 + except: + print(f"### {timestamp:012X} : {self.varnames[tag]}[{sub}] <= {value} [ERR] ", flush=True) + self.packets_dropped += 1 + elif datatag == 'S4': + timestamp = self.timestamp + # unpack + for i in range(0,4): + char = chr(value >> (i*8) & 0xFF) + self.strings[tag][1] += char + # sub of 1 indicates end of string + if sub == 1: + try: + #print(f"### {timestamp:012X} : {self.varnames[tag]} <= \"{self.strings[tag][1]}\"", flush=True) + self.writer.change(self.strings[tag][0], timestamp, self.strings[tag][1]) + except ValueError: + print(f"### {timestamp:012X} : {self.varnames[tag]} <= \"{self.strings[tag][1]}\" [VAL_ERR] ", flush=True) + self.packets_dropped += 1 + except writer.VCDPhaseError: + print(f"### {timestamp:012X} : {self.varnames[tag]} <= \"{self.strings[tag][1]}\" [PHA_ERR] ", flush=True) + self.packets_dropped += 1 + except: + print(f"### {timestamp:012X} : {self.varnames[tag]} <= \"{self.strings[tag][1]}\" [ERR] ", flush=True) + self.packets_dropped += 1 + self.strings[tag][1] = "" + + # skalar values + elif (datatag == 'EV') or (datatag[0] == 'V') or (datatag[0] == 'F'): + timestamp = self.timestamp + try: + if self.skalars[tag].type == VarType.event: + value = True + elif datatag == 'F4': + value = struct.unpack(">f", struct.pack(">L", value))[0] + #print(f"### {timestamp:012X} : {self.varnames[tag]} <= {value:08X}", flush=True) + self.writer.change(self.skalars[tag], timestamp, value) + except ValueError: + print(f"### {timestamp:012X} : {self.varnames[tag]} <= {value} [VAL_ERR] ", flush=True) + self.packets_dropped += 1 + except writer.VCDPhaseError: + print(f"### {timestamp:012X} : {self.varnames[tag]} <= {value} [PHA_ERR] ", flush=True) + self.packets_dropped += 1 + except: + print(f"### {timestamp:012X} : {self.varnames[tag]} <= {value} [ERR] ", flush=True) + self.packets_dropped += 1 + +def process_noise(noisefile, b): + print(chr(b), end="", flush=True) + if noisefile: + noisefile.write(b.to_bytes(1)) + +def main(): + parser = argparse.ArgumentParser(description="scans stdin for PET packets and dumps the values into a VCD file") + parser.add_argument('-d', '--dump', type=str, required=True, + help='output IEEE 1364-2005 Value Change Dump (vcd) file') + parser.add_argument('-t', '--timescale', type=str, default="1 us", + help='period of one timestamp tick') + parser.add_argument('-n', '--noise', type=str, default=None, + help='store the stdin data sans packets in this file') + parser.add_argument('-s', '--source', type=str, required=True, + help='source tree to scan for trace marks') + parser.add_argument('--diagnostics', action=argparse.BooleanOptionalAction, + help='add additional signals tracing internal state of PET') + args = parser.parse_args() + + print(header) + + tracefile = args.dump + noisefile = args.noise + source_tree = args.source + timescale = args.timescale + enable_diag = args.diagnostics + + predefined_signals = [] + if enable_diag: + predefined_signals += [ + 'PET.BufferItems:u32', + 'PET.BufferHealth:u8', + 'PET.CompressionLevel:u8', + 'PET.CompressionTime:u32', + 'PET.RenderTime:u32', + 'PET.ItemsSent:u32' + ] + + signals, signals_valid = scan_for_signals(source_tree, predefined_signals) + + if not signals_valid: + return + + signals.sort() + tags = [k.split(":")[0] for k in signals] + + dfile = open(tracefile, 'w', encoding='utf-8') + + process_noise_p = partial(process_noise, None) + nfile = None + if noisefile: + nfile = open(noisefile, 'wb') + process_noise_p = partial(process_noise, nfile) + + vcd_sink = VcdSink(dfile, signals, timescale) + retagger = Retagger(vcd_sink.process, tags) + packet_filter = Filter(retagger.process, process_noise_p) + print("Signals:") + for var in signals: + print(f" - {var}") + print() + + print(" === BEGIN NOISE ===") + try: + while True: + for b in sys.stdin.buffer.read(1): + packet_filter.process(b) + except KeyboardInterrupt: + pass + + print() + print(" === END NOISE ===") + print() + + vcd_sink.writer.close() + dfile.close() + if nfile: + nfile.close() + + print("Summary:") + packet_count = packet_filter.packet_counter + drop_count = packet_filter.packets_dropped + vcd_sink.packets_dropped + trace_size = human_readable_size(os.path.getsize(tracefile)) + print(f" - Packets received: {packet_count}") + print(f" - Packets dropped: {drop_count}") + print(f" - Trace file: {tracefile}") + print(f" - Trace size: {trace_size}") + +if __name__ == '__main__': + main()