add compression, change stream event format

...latest commit before the ESFM thing happened :D

oplplay/lxmplay/opmfile.h

@@ -69,11 +69,11 @@ enum {
     OPM_CMD00_SELECT_OPERATOR   = (1 << 5),
     OPM_CMD00_END_OF_FRAME      = (1 << 6),
 
-    OPM_CMD80_SET_KEYBLOCK      = (1 << 0),
+    OPM_CMD80_SET_TL0           = (1 << 0),
-    OPM_CMD80_SET_FREQ          = (1 << 1),
+    OPM_CMD80_SET_TL1           = (1 << 1),
     OPM_CMD80_SET_FEEDBACK      = (1 << 2),
-    OPM_CMD80_SET_TL0           = (1 << 3),
+    OPM_CMD80_SET_FREQ          = (1 << 3),
-    OPM_CMD80_SET_TL1           = (1 << 4),
+    OPM_CMD80_SET_KEYBLOCK      = (1 << 4),
     OPM_CMD80_END_OF_FRAME      = (1 << 5),
 
     OPM_CTRL_SET_REG01      = (1 << 0),

oplplay/lxmplay/opmplay.cpp

@@ -273,11 +273,11 @@ int opmplay_tick(opmplay_context_t* ctx, opl3_chip* opl3) {
                 if ((*(data) & 0xC0) == OPM_SET_FREQ_FB_VOL) {
                     int mask = *data;
                     data++;
-                    if (mask & OPM_CMD80_SET_KEYBLOCK)  { chctx->block = *data; OPL3_WriteRegBuffered(opl3, 0xB0 + ch, *data++); };
-                    if (mask & OPM_CMD80_SET_FREQ)      OPL3_WriteRegBuffered(opl3, 0xA0 + ch, *data++);
-                    if (mask & OPM_CMD80_SET_FEEDBACK)  OPL3_WriteRegBuffered(opl3, 0xC0 + ch, *data++);
                     if (mask & OPM_CMD80_SET_TL0)       OPL3_WriteRegBuffered(opl3, 0x40 + opmplay_opregs_channel_offset[ch], *data++);
                     if (mask & OPM_CMD80_SET_TL1)       OPL3_WriteRegBuffered(opl3, 0x43 + opmplay_opregs_channel_offset[ch], *data++);
+                    if (mask & OPM_CMD80_SET_FEEDBACK)  OPL3_WriteRegBuffered(opl3, 0xC0 + ch, *data++);
+                    if (mask & OPM_CMD80_SET_FREQ)      OPL3_WriteRegBuffered(opl3, 0xA0 + ch, *data++);
+                    if (mask & OPM_CMD80_SET_KEYBLOCK) { chctx->block = *data; OPL3_WriteRegBuffered(opl3, 0xB0 + ch, *data++); };
                     if (mask & OPM_CMD80_END_OF_FRAME)  isRun = false;
                     continue;
                 }
@@ -295,7 +295,7 @@ int opmplay_tick(opmplay_context_t* ctx, opl3_chip* opl3) {
                     chctx->stream.ptr = data + 3;
                     opmplay_push_stack(chctx);
                     data -= distance;
-                    chctx->stream.samples_to_play = frames_to_play;
+                    chctx->stream.samples_to_play = frames_to_play; // hack?
                     continue;
                 }
 

vgm2opl_next/vgm2opl_next/compressor.cpp

@@ -0,0 +1,191 @@
+#include <stdint.h>
+#include "opmctx.h"
+
+enum {
+    MAX_BACKREF_DISTANCE_FRAMES = 255,
+    MAX_BACKREF_DISTANCE_BYTES = 4095,
+    MAX_BACKREF_LENGTH_FRAMES = 255,
+};
+
+// -------------------------------------
+// compressor!
+
+struct match_info_t {
+    int logic_frames;       // nested backrefs count as 1 frame
+    int total_frames;       // incl. count in nested backrefs
+    int distance_frames;    // distance in frames
+    int distance_bytes;     // distance or length in bytes
+};
+
+// test current match, returns length or 0 if not matched, can be recursive
+static match_info_t test_match(
+    std::vector<opm_channel_record_t>& srcdata, std::vector<opm_channel_record_t>& window,
+    int datapos, int windowpos, int depth, int max_length = INT32_MAX
+) {
+    match_info_t rtn = { 0 };
+    if ((depth == 0) || (max_length == 0)) return rtn;
+
+    // start scanning from pos
+    int srcpos = datapos;
+    int dstpos = windowpos;
+    do {
+        auto src = srcdata.begin() + srcpos;
+        auto dst = window.begin() + dstpos;
+        // check for match
+        if ((src == srcdata.end()) || (dst == window.end())) break;
+        if (dst->flags & OPM_CHAN_BACKREF) {
+            // do recursive match
+            auto nested_match = test_match(srcdata, window, srcpos, dstpos - dst->distance_frames, depth - 1, dst->frames_to_play);
+            if (nested_match.logic_frames != dst->frames_to_play) break; else {
+                // add this backref to match count
+                dstpos++;
+                srcpos += nested_match.logic_frames;
+                rtn.logic_frames++;
+                rtn.total_frames += nested_match.logic_frames;
+            }
+        }
+        else {
+            // no backref, scan current frame
+            if ((src->rawdata != dst->rawdata) || (src->frame_dist != dst->frame_dist) ||
+                (rtn.distance_bytes + src->rawdata.size() >= MAX_BACKREF_DISTANCE_BYTES))
+            {
+                // match failure or too long - enough for us :)
+                break;
+            } else {
+                // match! increment everything
+                srcpos++; dstpos++;
+                rtn.logic_frames++;
+                rtn.total_frames++;
+                rtn.distance_bytes += src->rawdata.size();
+            }
+        }
+    } while (--max_length != 0);
+
+    // validate
+    if ((rtn.distance_bytes > MAX_BACKREF_DISTANCE_BYTES) ||
+        (rtn.logic_frames > MAX_BACKREF_DISTANCE_FRAMES)) return { 0 };
+
+    return rtn;
+}
+
+// find match
+static match_info_t find_match(opm_convert_context_t *ctx,
+    std::vector<opm_channel_record_t>& srcdata, std::vector<opm_channel_record_t>& dstdata,
+    int current_pos, int max_lookback, int min_backref_length
+) {
+    match_info_t ret = { 0 };
+
+    // start from back
+    int start = dstdata.size() - max_lookback; if (start < 0) start = 0;
+    int stop = srcdata.size();
+
+    for (int pos = start; pos < dstdata.size(); pos++) {
+        int srcpos = current_pos;
+        int dstpos = pos;
+
+        // test current match
+        auto match = test_match(srcdata, dstdata, srcpos, dstpos, ctx->flags.max_stack_depth, MAX_BACKREF_LENGTH_FRAMES);
+
+        // check if match is found
+        if (match.total_frames >= min_backref_length) {
+            // emit match and end
+            ret.distance_frames = dstdata.size() - pos;
+            ret.logic_frames = match.logic_frames;
+            ret.total_frames = match.total_frames;
+            return ret;
+        }
+    }
+
+    return {};
+}
+
+uint32_t opm_compress_channel(opm_convert_context_t* ctx, int ch, int min_backref_length) {
+    ctx->opmpacked[ch].clear();
+    auto& src_ch = ctx->opmrecords[ch];
+    int pos = 0; uint32_t total_bytes = 0;
+    while (pos != src_ch.size()) {
+        // find match
+        auto match = find_match(ctx, src_ch, ctx->opmpacked[ch], pos, MAX_BACKREF_DISTANCE_FRAMES, min_backref_length);
+        if (match.logic_frames == 0) {
+            // copy one frame
+            total_bytes += src_ch[pos].rawdata.size();
+            ctx->opmpacked[ch].push_back(src_ch[pos]);
+            pos++;
+        }
+        else {
+            // set back reference - copy main parameters from original
+            opm_channel_record_t rec = src_ch[pos];
+            rec.flags |= OPM_CHAN_BACKREF;
+            rec.distance_frames = match.distance_frames;
+            rec.frames_to_play = match.logic_frames;
+            // fill dummy rawdata (will resolve this later!)
+            rec.rawdata.clear();
+            rec.rawdata.push_back(OPM_STREAM_BACKREF);
+            rec.rawdata.push_back(0);
+            rec.rawdata.push_back(rec.frames_to_play);
+            ctx->opmpacked[ch].push_back(rec);
+            pos += match.total_frames; // skip matched data
+            total_bytes += rec.rawdata.size();
+        }
+    }
+    if (ctx->flags.verbosity >= 2) {
+        printf("ch %d min backref %d - %d records, %d bytes\n", ch, min_backref_length, ctx->opmpacked[ch].size(), total_bytes);
+    }
+    return total_bytes;
+}
+
+void opm_compress(opm_convert_context_t* ctx) {
+    ctx->opmpacked.resize(ctx->opmrecords.size());
+    printf("compressing"); fflush(stdout);
+
+    // messy backref bruteforce stuff
+    struct backref_bruteforce_t {
+        int min, max;
+        int best; uint32_t bestsize;
+    } backref_len;
+    switch (ctx->flags.compress_level) {
+    case 2: backref_len.min = 2; backref_len.max = 16; break;      // bruteforced
+    case 1:
+    default: backref_len.min = backref_len.max = 4; break;         // fixed 
+    };
+
+    int ch = 0;
+    for (auto& src_ch : ctx->opmrecords) {
+        if (ctx->flags.verbosity >= 2) {
+            printf("\n", ch);
+        }
+        backref_bruteforce_t cur_backref_len;
+        cur_backref_len.min = backref_len.min;
+        cur_backref_len.max = backref_len.max;
+        cur_backref_len.best = backref_len.min;
+        cur_backref_len.bestsize = -1;
+        int sz;
+        for (sz = cur_backref_len.min; sz <= cur_backref_len.max; sz++) {
+            auto bytes = opm_compress_channel(ctx, ch, sz);
+            if (cur_backref_len.bestsize >= bytes) {
+                cur_backref_len.bestsize = bytes;
+                cur_backref_len.best = sz;
+            }
+        }
+        // recompress if current != best
+        if (sz-1 != cur_backref_len.best) opm_compress_channel(ctx, ch, cur_backref_len.best);
+        // resolve back references
+        std::vector<uint32_t> ch_bytepos;
+        uint32_t bytepos_cur = 0;
+        for (int f = 0; f < ctx->opmpacked[ch].size(); f++) {
+            ch_bytepos.push_back(bytepos_cur);
+            // fixup back reference (if any)
+            if (ctx->opmpacked[ch][f].flags & OPM_CHAN_BACKREF) {
+                uint32_t backref_dist = bytepos_cur - ch_bytepos[f - ctx->opmpacked[ch][f].distance_frames];
+                ctx->opmpacked[ch][f].rawdata[0] = OPM_STREAM_BACKREF | (backref_dist >> 8);
+                ctx->opmpacked[ch][f].rawdata[1] = (backref_dist & 0xFF);
+            }
+            bytepos_cur += ctx->opmpacked[ch][f].rawdata.size();
+        }
+        ch++;
+#ifndef ULTRA_DEBUG
+        printf("."); fflush(stdout);
+#endif
+    }
+    printf("done\n");
+}

vgm2opl_next/vgm2opl_next/compressor.h

@@ -0,0 +1,7 @@
+#pragma once
+
+#include <stdint.h>
+#include "opmctx.h"
+
+// main compression routine
+void opm_compress(opm_convert_context_t* ctx);

vgm2opl_next/vgm2opl_next/opmctx.h

@@ -36,9 +36,9 @@ enum {
     OPM_WAVEFORM        = (1 << 4),
     OPM_OP1_SHIFT       = 5,
 
-    OPM_BLOCK           = (1 << 10),
+    OPM_FEEDBACK        = (1 << 10),
     OPM_FNUM            = (1 << 11),
-    OPM_FEEDBACK        = (1 << 12),
+    OPM_BLOCK           = (1 << 12),
     OPM_KEY             = (1 << 13),
 
     OPM_KEYPERC         = (1 << 14),
@@ -49,8 +49,8 @@ enum {
     OPM_REG_BD          = (1 << 19),
 
     // opm_serialize_channel_stream() tweaks
-    OPM_TL0             = (1 << 13),
+    OPM_TL0             = (1 << 8),
-    OPM_TL1             = (1 << 14),
+    OPM_TL1             = (1 << 9),
 };
 
 struct opm_frame_record {
@@ -97,7 +97,7 @@ struct opm_channel_record_t {
     // records
     std::vector<opm_frame_record> records;
     // compression data
-    int distance, frames_to_play, frames_to_play_total;
+    int distance_frames, distance_bytes, frames_to_play, frames_to_play_total;
 };
 
 
@@ -133,6 +133,7 @@ struct opm_convert_context_t {
     struct {
         int  compress_level;
         int  max_stack_depth;
+        int  verbosity;
     } flags;
 
     // frame rate stuff

vgm2opl_next/vgm2opl_next/opmfile.h

@@ -69,11 +69,11 @@ enum {
     OPM_CMD00_SELECT_OPERATOR   = (1 << 5),
     OPM_CMD00_END_OF_FRAME      = (1 << 6),
 
-    OPM_CMD80_SET_KEYBLOCK      = (1 << 0),
+    OPM_CMD80_SET_TL0           = (1 << 0),
-    OPM_CMD80_SET_FREQ          = (1 << 1),
+    OPM_CMD80_SET_TL1           = (1 << 1),
     OPM_CMD80_SET_FEEDBACK      = (1 << 2),
-    OPM_CMD80_SET_TL0           = (1 << 3),
+    OPM_CMD80_SET_FREQ          = (1 << 3),
-    OPM_CMD80_SET_TL1           = (1 << 4),
+    OPM_CMD80_SET_KEYBLOCK      = (1 << 4),
     OPM_CMD80_END_OF_FRAME      = (1 << 5),
 
     OPM_CTRL_SET_REG01      = (1 << 0),

vgm2opl_next/vgm2opl_next/vgm2opl_next.cpp

@@ -10,6 +10,7 @@
 #include "opmfile.h"
 #include "opmctx.h"
 #include "vgm.h"
+#include "compressor.h"
 
 opm_convert_context_t convert_ctx;
 opm_convert_context_t* ctx = &convert_ctx;
@@ -647,14 +648,14 @@ int opm_serialize_channel_stream(opm_convert_context_t* ctx, int ch) {
                 // freq/feedback
                 if ((mask.last & 1) && !mask.key_only) {
                     s.rawdata.push_back(
-                        OPM_SET_FREQ_FB_VOL | (mask.freq_fb >> 10) |
+                        OPM_SET_FREQ_FB_VOL | (mask.freq_fb >> 8) |
                         (mask.last == 1 ? OPM_CMD80_END_OF_FRAME : 0)
                     );
-                    if (mask.freq_fb & OPM_BLOCK)       s.rawdata.push_back(e.block);
-                    if (mask.freq_fb & OPM_FNUM)        s.rawdata.push_back(e.fnum);
-                    if (mask.freq_fb & OPM_FEEDBACK)    s.rawdata.push_back(e.feedback);
                     if (mask.freq_fb & OPM_TL0)         s.rawdata.push_back(e.op[0].ksl_tl);
                     if (mask.freq_fb & OPM_TL1)         s.rawdata.push_back(e.op[1].ksl_tl);
+                    if (mask.freq_fb & OPM_FEEDBACK)    s.rawdata.push_back(e.feedback);
+                    if (mask.freq_fb & OPM_FNUM)        s.rawdata.push_back(e.fnum);
+                    if (mask.freq_fb & OPM_BLOCK)       s.rawdata.push_back(e.block);
                 }
                 mask.last >>= 1;
                 if (mask.key_only) {
@@ -687,7 +688,8 @@ int opm_serialize_stream(opm_convert_context_t* ctx) {
 int opm_concat_streams(opm_convert_context_t* ctx) {
     ctx->oplchan_out.resize(1 + 9);     // TODO: FIXME for OPL3 support!!
     for (int ch = 0; ch < (1 + 9); ch++) {
-        for (auto& f : ctx->opmrecords[ch]) {
+        //for (auto& f : ctx->opmrecords[ch]) {
+        for (auto& f : ctx->opmpacked[ch]) {
             // copy raw register data
             ctx->oplchan_out[ch].insert(ctx->oplchan_out[ch].end(), f.rawdata.begin(), f.rawdata.end());
         }
@@ -706,31 +708,55 @@ int opm_dump_events(opm_convert_context_t* ctx) {
     for (int ch = 0; ch < (1 + 9); ch++) {
         fprintf(f, "---channel %d\n", ch-1);
         for (auto& a : ctx->opmrecords[ch]) {
-            fprintf(f, "frame %07d, dist %d: data ", a.frame, a.frame_dist, a.records.size());
+            fprintf(f, "frame %07d, dist %d: data ", a.frame, a.frame_dist);
             for (auto& d : a.rawdata) fprintf(f, "%02X ", d);
             fprintf(f,"\n");
         }
     }
     fclose(f);
+
+    f = fopen((ctx->logname+".packed.log").c_str(), "w");
+    fprintf(f, "total %d frames\n", ctx->total_frames);
+    for (int ch = 0; ch < (1 + 9); ch++) {
+        fprintf(f, "---channel %d\n", ch - 1);
+        for (auto& a : ctx->opmpacked[ch]) {
+            fprintf(f, "frame %07d, dist %d: data ", a.frame, a.frame_dist);
+            for (auto& d : a.rawdata) fprintf(f, "%02X ", d);
+            fprintf(f, "\n");
+        }
+    }
+    fclose(f);
     return 0;
 }
 
+// -------------------------
+
+const cmdline_t cmdparams[] = {
+    {'C',   CMD_FLAG_INT,      "COMPRESSION",       &ctx->flags.compress_level},
+    {'D',   CMD_FLAG_INT,      "DEPTH",             &ctx->flags.max_stack_depth},
+    {'V',   CMD_FLAG_INT,      "VERBOSITY",         &ctx->flags.verbosity},
+};
+
 // ----------------
 
 int main(int argc, char* argv[]) {
     // clear compression context!
-    ctx->flags.compress_level = 0;
+    ctx->flags.compress_level   = 1;
+    ctx->flags.max_stack_depth  = 2;
+    ctx->flags.verbosity        = 1;
 
     if (argc < 2) {
-        printf("usage: vgm2opl input.vgm [-px]\n");
+        printf("usage: vgm2opm input.vgm [-cx] [-dx] [-vx]\n");
-        printf("-p[x]: enable compression, [x] - mode:\n"\
+        printf("-c[x]: enable compression, [x] - mode (default - 1):\n");
-            "       1 - fixed min backref, 2 - bruteforce best backref\n");
+        printf("       0 - disabled, 1 - fixed min backref, 2 - bruteforce best backref\n");
+        printf("-d[x]: maximum back reference stack depth (default - 2)\n");
+        printf("-v[x]: verbosity level (default - 1)\n");
         return 1;
     }
 
     // read parameters
-#if 0
+#if 1
-    if (parse_cmdline(argc, argv, cmdparams, 1, 2) != 0) {
+    if (parse_cmdline(argc, argv, cmdparams, 3, 2) != 0) {
         printf("error: unable to parse command line!\n");
         return 1;
     }
@@ -770,24 +796,17 @@ int main(int argc, char* argv[]) {
     if (ctx->vgm.header->loopOffset != 0) ctx->vgm.loop_pos = ctx->vgm.header->loopOffset + offsetof(VGMHeader, loopOffset);
     ctx->vgm.end = ctx->vgm.header->eofOffset + offsetof(VGMHeader, eofOffset);
     ctx->vgm.start = ((ctx->vgm.header->version < 0x150) ? 0x40 : ctx->vgm.header->dataOffset + offsetof(VGMHeader, dataOffset));
-    
+
-#if 0
+    auto oplClockRate = std::max(ctx->vgm.header->YM3812_Clock, ctx->vgm.header->YMF262_Clock);
+    printf("frame rate = %d Hz, OPL clock rate = %d Hz\n", ctx->vgm.header->frameRate, oplClockRate);
+
     // check if OPL2/3 is present
-    if (vgmhead->YM3812_Clock == 0) {
+    if (oplClockRate == 0) {
-        printf("OPL2 data not found!\n");
+        printf("OPL2/3 data not found!\n");
         return 1;
     }
-#endif
 
-    printf("frame rate = %d Hz, OPL clock rate = %d Hz\n", ctx->vgm.header->frameRate, std::max(ctx->vgm.header->YM3812_Clock, ctx->vgm.header->YMF262_Clock));
+    // set estimation parameters
-
-#if 0
-    // estimate frame rate
-    uint32_t framerate = vgm_estimate_framerate(vgmfile, vgmdata, vgmend);
-    printf("estimated frame rate = %d samples [%.3f Hz]\n", framerate, 44100.0 / framerate);
-#endif
-
-    // set estimate parameters
     ctx->estimate.max_delay_base = (44100.0 / ((double)0x1234DD / 65536));
     ctx->estimate.max_delay_freq = 400.0;
     ctx->estimate.trim_threshold = 0.005;
@@ -807,13 +826,11 @@ int main(int argc, char* argv[]) {
     // serialize to sequence of bytes
     opm_serialize_stream(ctx);
 
-#if 0
+    // coplress the stream
-    // generate simpel stream
+    opm_compress(ctx);
-    opm_dump_simple(ctx);
+
-#else 
     // concatenate streams
     opm_concat_streams(ctx);
-#endif
 
     // dump raw data
     opm_dump_events(ctx);
@@ -821,6 +838,9 @@ int main(int argc, char* argv[]) {
     // dump OPM file
     opm_write_file(ctx);
 
+    // write statistics
+    // TODO
+
     printf("done\n");
 
     return 0;

vgm2opl_next/vgm2opl_next/vgm2opl_next.vcxproj

@@ -88,6 +88,7 @@
       <SDLCheck>true</SDLCheck>
       <PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <ConformanceMode>true</ConformanceMode>
+      <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
     </ClCompile>
     <Link>
       <SubSystem>Console</SubSystem>
@@ -102,6 +103,7 @@
       <SDLCheck>true</SDLCheck>
       <PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <ConformanceMode>true</ConformanceMode>
+      <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
     </ClCompile>
     <Link>
       <SubSystem>Console</SubSystem>
@@ -140,10 +142,12 @@
   </ItemDefinitionGroup>
   <ItemGroup>
     <ClCompile Include="cmdline.cpp" />
+    <ClCompile Include="compressor.cpp" />
     <ClCompile Include="vgm2opl_next.cpp" />
   </ItemGroup>
   <ItemGroup>
     <ClInclude Include="cmdline.h" />
+    <ClInclude Include="compressor.h" />
     <ClInclude Include="opmctx.h" />
     <ClInclude Include="opmfile.h" />
     <ClInclude Include="vgm.h" />

vgm2opl_next/vgm2opl_next/vgm2opl_next.vcxproj.filters

@@ -21,6 +21,9 @@
     <ClCompile Include="cmdline.cpp">
       <Filter>Исходные файлы</Filter>
     </ClCompile>
+    <ClCompile Include="compressor.cpp">
+      <Filter>Исходные файлы</Filter>
+    </ClCompile>
   </ItemGroup>
   <ItemGroup>
     <ClInclude Include="vgm.h">
@@ -35,5 +38,8 @@
     <ClInclude Include="cmdline.h">
       <Filter>Файлы заголовков</Filter>
     </ClInclude>
+    <ClInclude Include="compressor.h">
+      <Filter>Файлы заголовков</Filter>
+    </ClInclude>
   </ItemGroup>
 </Project>