add compression, change stream event format

...latest commit before the ESFM thing happened :D
This commit is contained in:
wbcbz7 2024-06-06 01:36:14 +07:00
parent a681a2c53b
commit 6fca400cdd
9 changed files with 277 additions and 48 deletions

View file

@ -69,11 +69,11 @@ enum {
OPM_CMD00_SELECT_OPERATOR = (1 << 5),
OPM_CMD00_END_OF_FRAME = (1 << 6),
OPM_CMD80_SET_KEYBLOCK = (1 << 0),
OPM_CMD80_SET_FREQ = (1 << 1),
OPM_CMD80_SET_TL0 = (1 << 0),
OPM_CMD80_SET_TL1 = (1 << 1),
OPM_CMD80_SET_FEEDBACK = (1 << 2),
OPM_CMD80_SET_TL0 = (1 << 3),
OPM_CMD80_SET_TL1 = (1 << 4),
OPM_CMD80_SET_FREQ = (1 << 3),
OPM_CMD80_SET_KEYBLOCK = (1 << 4),
OPM_CMD80_END_OF_FRAME = (1 << 5),
OPM_CTRL_SET_REG01 = (1 << 0),

View file

@ -273,11 +273,11 @@ int opmplay_tick(opmplay_context_t* ctx, opl3_chip* opl3) {
if ((*(data) & 0xC0) == OPM_SET_FREQ_FB_VOL) {
int mask = *data;
data++;
if (mask & OPM_CMD80_SET_KEYBLOCK) { chctx->block = *data; OPL3_WriteRegBuffered(opl3, 0xB0 + ch, *data++); };
if (mask & OPM_CMD80_SET_FREQ) OPL3_WriteRegBuffered(opl3, 0xA0 + ch, *data++);
if (mask & OPM_CMD80_SET_FEEDBACK) OPL3_WriteRegBuffered(opl3, 0xC0 + ch, *data++);
if (mask & OPM_CMD80_SET_TL0) OPL3_WriteRegBuffered(opl3, 0x40 + opmplay_opregs_channel_offset[ch], *data++);
if (mask & OPM_CMD80_SET_TL1) OPL3_WriteRegBuffered(opl3, 0x43 + opmplay_opregs_channel_offset[ch], *data++);
if (mask & OPM_CMD80_SET_FEEDBACK) OPL3_WriteRegBuffered(opl3, 0xC0 + ch, *data++);
if (mask & OPM_CMD80_SET_FREQ) OPL3_WriteRegBuffered(opl3, 0xA0 + ch, *data++);
if (mask & OPM_CMD80_SET_KEYBLOCK) { chctx->block = *data; OPL3_WriteRegBuffered(opl3, 0xB0 + ch, *data++); };
if (mask & OPM_CMD80_END_OF_FRAME) isRun = false;
continue;
}
@ -295,7 +295,7 @@ int opmplay_tick(opmplay_context_t* ctx, opl3_chip* opl3) {
chctx->stream.ptr = data + 3;
opmplay_push_stack(chctx);
data -= distance;
chctx->stream.samples_to_play = frames_to_play;
chctx->stream.samples_to_play = frames_to_play; // hack?
continue;
}

View file

@ -0,0 +1,191 @@
#include <stdint.h>
#include "opmctx.h"
enum {
MAX_BACKREF_DISTANCE_FRAMES = 255,
MAX_BACKREF_DISTANCE_BYTES = 4095,
MAX_BACKREF_LENGTH_FRAMES = 255,
};
// -------------------------------------
// compressor!
struct match_info_t {
int logic_frames; // nested backrefs count as 1 frame
int total_frames; // incl. count in nested backrefs
int distance_frames; // distance in frames
int distance_bytes; // distance or length in bytes
};
// test current match, returns length or 0 if not matched, can be recursive
static match_info_t test_match(
std::vector<opm_channel_record_t>& srcdata, std::vector<opm_channel_record_t>& window,
int datapos, int windowpos, int depth, int max_length = INT32_MAX
) {
match_info_t rtn = { 0 };
if ((depth == 0) || (max_length == 0)) return rtn;
// start scanning from pos
int srcpos = datapos;
int dstpos = windowpos;
do {
auto src = srcdata.begin() + srcpos;
auto dst = window.begin() + dstpos;
// check for match
if ((src == srcdata.end()) || (dst == window.end())) break;
if (dst->flags & OPM_CHAN_BACKREF) {
// do recursive match
auto nested_match = test_match(srcdata, window, srcpos, dstpos - dst->distance_frames, depth - 1, dst->frames_to_play);
if (nested_match.logic_frames != dst->frames_to_play) break; else {
// add this backref to match count
dstpos++;
srcpos += nested_match.logic_frames;
rtn.logic_frames++;
rtn.total_frames += nested_match.logic_frames;
}
}
else {
// no backref, scan current frame
if ((src->rawdata != dst->rawdata) || (src->frame_dist != dst->frame_dist) ||
(rtn.distance_bytes + src->rawdata.size() >= MAX_BACKREF_DISTANCE_BYTES))
{
// match failure or too long - enough for us :)
break;
} else {
// match! increment everything
srcpos++; dstpos++;
rtn.logic_frames++;
rtn.total_frames++;
rtn.distance_bytes += src->rawdata.size();
}
}
} while (--max_length != 0);
// validate
if ((rtn.distance_bytes > MAX_BACKREF_DISTANCE_BYTES) ||
(rtn.logic_frames > MAX_BACKREF_DISTANCE_FRAMES)) return { 0 };
return rtn;
}
// find match
static match_info_t find_match(opm_convert_context_t *ctx,
std::vector<opm_channel_record_t>& srcdata, std::vector<opm_channel_record_t>& dstdata,
int current_pos, int max_lookback, int min_backref_length
) {
match_info_t ret = { 0 };
// start from back
int start = dstdata.size() - max_lookback; if (start < 0) start = 0;
int stop = srcdata.size();
for (int pos = start; pos < dstdata.size(); pos++) {
int srcpos = current_pos;
int dstpos = pos;
// test current match
auto match = test_match(srcdata, dstdata, srcpos, dstpos, ctx->flags.max_stack_depth, MAX_BACKREF_LENGTH_FRAMES);
// check if match is found
if (match.total_frames >= min_backref_length) {
// emit match and end
ret.distance_frames = dstdata.size() - pos;
ret.logic_frames = match.logic_frames;
ret.total_frames = match.total_frames;
return ret;
}
}
return {};
}
uint32_t opm_compress_channel(opm_convert_context_t* ctx, int ch, int min_backref_length) {
ctx->opmpacked[ch].clear();
auto& src_ch = ctx->opmrecords[ch];
int pos = 0; uint32_t total_bytes = 0;
while (pos != src_ch.size()) {
// find match
auto match = find_match(ctx, src_ch, ctx->opmpacked[ch], pos, MAX_BACKREF_DISTANCE_FRAMES, min_backref_length);
if (match.logic_frames == 0) {
// copy one frame
total_bytes += src_ch[pos].rawdata.size();
ctx->opmpacked[ch].push_back(src_ch[pos]);
pos++;
}
else {
// set back reference - copy main parameters from original
opm_channel_record_t rec = src_ch[pos];
rec.flags |= OPM_CHAN_BACKREF;
rec.distance_frames = match.distance_frames;
rec.frames_to_play = match.logic_frames;
// fill dummy rawdata (will resolve this later!)
rec.rawdata.clear();
rec.rawdata.push_back(OPM_STREAM_BACKREF);
rec.rawdata.push_back(0);
rec.rawdata.push_back(rec.frames_to_play);
ctx->opmpacked[ch].push_back(rec);
pos += match.total_frames; // skip matched data
total_bytes += rec.rawdata.size();
}
}
if (ctx->flags.verbosity >= 2) {
printf("ch %d min backref %d - %d records, %d bytes\n", ch, min_backref_length, ctx->opmpacked[ch].size(), total_bytes);
}
return total_bytes;
}
void opm_compress(opm_convert_context_t* ctx) {
ctx->opmpacked.resize(ctx->opmrecords.size());
printf("compressing"); fflush(stdout);
// messy backref bruteforce stuff
struct backref_bruteforce_t {
int min, max;
int best; uint32_t bestsize;
} backref_len;
switch (ctx->flags.compress_level) {
case 2: backref_len.min = 2; backref_len.max = 16; break; // bruteforced
case 1:
default: backref_len.min = backref_len.max = 4; break; // fixed
};
int ch = 0;
for (auto& src_ch : ctx->opmrecords) {
if (ctx->flags.verbosity >= 2) {
printf("\n", ch);
}
backref_bruteforce_t cur_backref_len;
cur_backref_len.min = backref_len.min;
cur_backref_len.max = backref_len.max;
cur_backref_len.best = backref_len.min;
cur_backref_len.bestsize = -1;
int sz;
for (sz = cur_backref_len.min; sz <= cur_backref_len.max; sz++) {
auto bytes = opm_compress_channel(ctx, ch, sz);
if (cur_backref_len.bestsize >= bytes) {
cur_backref_len.bestsize = bytes;
cur_backref_len.best = sz;
}
}
// recompress if current != best
if (sz-1 != cur_backref_len.best) opm_compress_channel(ctx, ch, cur_backref_len.best);
// resolve back references
std::vector<uint32_t> ch_bytepos;
uint32_t bytepos_cur = 0;
for (int f = 0; f < ctx->opmpacked[ch].size(); f++) {
ch_bytepos.push_back(bytepos_cur);
// fixup back reference (if any)
if (ctx->opmpacked[ch][f].flags & OPM_CHAN_BACKREF) {
uint32_t backref_dist = bytepos_cur - ch_bytepos[f - ctx->opmpacked[ch][f].distance_frames];
ctx->opmpacked[ch][f].rawdata[0] = OPM_STREAM_BACKREF | (backref_dist >> 8);
ctx->opmpacked[ch][f].rawdata[1] = (backref_dist & 0xFF);
}
bytepos_cur += ctx->opmpacked[ch][f].rawdata.size();
}
ch++;
#ifndef ULTRA_DEBUG
printf("."); fflush(stdout);
#endif
}
printf("done\n");
}

View file

@ -0,0 +1,7 @@
#pragma once
#include <stdint.h>
#include "opmctx.h"
// main compression routine
void opm_compress(opm_convert_context_t* ctx);

View file

@ -36,9 +36,9 @@ enum {
OPM_WAVEFORM = (1 << 4),
OPM_OP1_SHIFT = 5,
OPM_BLOCK = (1 << 10),
OPM_FEEDBACK = (1 << 10),
OPM_FNUM = (1 << 11),
OPM_FEEDBACK = (1 << 12),
OPM_BLOCK = (1 << 12),
OPM_KEY = (1 << 13),
OPM_KEYPERC = (1 << 14),
@ -49,8 +49,8 @@ enum {
OPM_REG_BD = (1 << 19),
// opm_serialize_channel_stream() tweaks
OPM_TL0 = (1 << 13),
OPM_TL1 = (1 << 14),
OPM_TL0 = (1 << 8),
OPM_TL1 = (1 << 9),
};
struct opm_frame_record {
@ -97,7 +97,7 @@ struct opm_channel_record_t {
// records
std::vector<opm_frame_record> records;
// compression data
int distance, frames_to_play, frames_to_play_total;
int distance_frames, distance_bytes, frames_to_play, frames_to_play_total;
};
@ -133,6 +133,7 @@ struct opm_convert_context_t {
struct {
int compress_level;
int max_stack_depth;
int verbosity;
} flags;
// frame rate stuff

View file

@ -69,11 +69,11 @@ enum {
OPM_CMD00_SELECT_OPERATOR = (1 << 5),
OPM_CMD00_END_OF_FRAME = (1 << 6),
OPM_CMD80_SET_KEYBLOCK = (1 << 0),
OPM_CMD80_SET_FREQ = (1 << 1),
OPM_CMD80_SET_TL0 = (1 << 0),
OPM_CMD80_SET_TL1 = (1 << 1),
OPM_CMD80_SET_FEEDBACK = (1 << 2),
OPM_CMD80_SET_TL0 = (1 << 3),
OPM_CMD80_SET_TL1 = (1 << 4),
OPM_CMD80_SET_FREQ = (1 << 3),
OPM_CMD80_SET_KEYBLOCK = (1 << 4),
OPM_CMD80_END_OF_FRAME = (1 << 5),
OPM_CTRL_SET_REG01 = (1 << 0),

View file

@ -10,6 +10,7 @@
#include "opmfile.h"
#include "opmctx.h"
#include "vgm.h"
#include "compressor.h"
opm_convert_context_t convert_ctx;
opm_convert_context_t* ctx = &convert_ctx;
@ -647,14 +648,14 @@ int opm_serialize_channel_stream(opm_convert_context_t* ctx, int ch) {
// freq/feedback
if ((mask.last & 1) && !mask.key_only) {
s.rawdata.push_back(
OPM_SET_FREQ_FB_VOL | (mask.freq_fb >> 10) |
OPM_SET_FREQ_FB_VOL | (mask.freq_fb >> 8) |
(mask.last == 1 ? OPM_CMD80_END_OF_FRAME : 0)
);
if (mask.freq_fb & OPM_BLOCK) s.rawdata.push_back(e.block);
if (mask.freq_fb & OPM_FNUM) s.rawdata.push_back(e.fnum);
if (mask.freq_fb & OPM_FEEDBACK) s.rawdata.push_back(e.feedback);
if (mask.freq_fb & OPM_TL0) s.rawdata.push_back(e.op[0].ksl_tl);
if (mask.freq_fb & OPM_TL1) s.rawdata.push_back(e.op[1].ksl_tl);
if (mask.freq_fb & OPM_FEEDBACK) s.rawdata.push_back(e.feedback);
if (mask.freq_fb & OPM_FNUM) s.rawdata.push_back(e.fnum);
if (mask.freq_fb & OPM_BLOCK) s.rawdata.push_back(e.block);
}
mask.last >>= 1;
if (mask.key_only) {
@ -687,7 +688,8 @@ int opm_serialize_stream(opm_convert_context_t* ctx) {
int opm_concat_streams(opm_convert_context_t* ctx) {
ctx->oplchan_out.resize(1 + 9); // TODO: FIXME for OPL3 support!!
for (int ch = 0; ch < (1 + 9); ch++) {
for (auto& f : ctx->opmrecords[ch]) {
//for (auto& f : ctx->opmrecords[ch]) {
for (auto& f : ctx->opmpacked[ch]) {
// copy raw register data
ctx->oplchan_out[ch].insert(ctx->oplchan_out[ch].end(), f.rawdata.begin(), f.rawdata.end());
}
@ -706,31 +708,55 @@ int opm_dump_events(opm_convert_context_t* ctx) {
for (int ch = 0; ch < (1 + 9); ch++) {
fprintf(f, "---channel %d\n", ch-1);
for (auto& a : ctx->opmrecords[ch]) {
fprintf(f, "frame %07d, dist %d: data ", a.frame, a.frame_dist, a.records.size());
fprintf(f, "frame %07d, dist %d: data ", a.frame, a.frame_dist);
for (auto& d : a.rawdata) fprintf(f, "%02X ", d);
fprintf(f,"\n");
}
}
fclose(f);
f = fopen((ctx->logname+".packed.log").c_str(), "w");
fprintf(f, "total %d frames\n", ctx->total_frames);
for (int ch = 0; ch < (1 + 9); ch++) {
fprintf(f, "---channel %d\n", ch - 1);
for (auto& a : ctx->opmpacked[ch]) {
fprintf(f, "frame %07d, dist %d: data ", a.frame, a.frame_dist);
for (auto& d : a.rawdata) fprintf(f, "%02X ", d);
fprintf(f, "\n");
}
}
fclose(f);
return 0;
}
// -------------------------
const cmdline_t cmdparams[] = {
{'C', CMD_FLAG_INT, "COMPRESSION", &ctx->flags.compress_level},
{'D', CMD_FLAG_INT, "DEPTH", &ctx->flags.max_stack_depth},
{'V', CMD_FLAG_INT, "VERBOSITY", &ctx->flags.verbosity},
};
// ----------------
int main(int argc, char* argv[]) {
// clear compression context!
ctx->flags.compress_level = 0;
ctx->flags.compress_level = 1;
ctx->flags.max_stack_depth = 2;
ctx->flags.verbosity = 1;
if (argc < 2) {
printf("usage: vgm2opl input.vgm [-px]\n");
printf("-p[x]: enable compression, [x] - mode:\n"\
" 1 - fixed min backref, 2 - bruteforce best backref\n");
printf("usage: vgm2opm input.vgm [-cx] [-dx] [-vx]\n");
printf("-c[x]: enable compression, [x] - mode (default - 1):\n");
printf(" 0 - disabled, 1 - fixed min backref, 2 - bruteforce best backref\n");
printf("-d[x]: maximum back reference stack depth (default - 2)\n");
printf("-v[x]: verbosity level (default - 1)\n");
return 1;
}
// read parameters
#if 0
if (parse_cmdline(argc, argv, cmdparams, 1, 2) != 0) {
#if 1
if (parse_cmdline(argc, argv, cmdparams, 3, 2) != 0) {
printf("error: unable to parse command line!\n");
return 1;
}
@ -770,24 +796,17 @@ int main(int argc, char* argv[]) {
if (ctx->vgm.header->loopOffset != 0) ctx->vgm.loop_pos = ctx->vgm.header->loopOffset + offsetof(VGMHeader, loopOffset);
ctx->vgm.end = ctx->vgm.header->eofOffset + offsetof(VGMHeader, eofOffset);
ctx->vgm.start = ((ctx->vgm.header->version < 0x150) ? 0x40 : ctx->vgm.header->dataOffset + offsetof(VGMHeader, dataOffset));
#if 0
auto oplClockRate = std::max(ctx->vgm.header->YM3812_Clock, ctx->vgm.header->YMF262_Clock);
printf("frame rate = %d Hz, OPL clock rate = %d Hz\n", ctx->vgm.header->frameRate, oplClockRate);
// check if OPL2/3 is present
if (vgmhead->YM3812_Clock == 0) {
printf("OPL2 data not found!\n");
if (oplClockRate == 0) {
printf("OPL2/3 data not found!\n");
return 1;
}
#endif
printf("frame rate = %d Hz, OPL clock rate = %d Hz\n", ctx->vgm.header->frameRate, std::max(ctx->vgm.header->YM3812_Clock, ctx->vgm.header->YMF262_Clock));
#if 0
// estimate frame rate
uint32_t framerate = vgm_estimate_framerate(vgmfile, vgmdata, vgmend);
printf("estimated frame rate = %d samples [%.3f Hz]\n", framerate, 44100.0 / framerate);
#endif
// set estimate parameters
// set estimation parameters
ctx->estimate.max_delay_base = (44100.0 / ((double)0x1234DD / 65536));
ctx->estimate.max_delay_freq = 400.0;
ctx->estimate.trim_threshold = 0.005;
@ -807,13 +826,11 @@ int main(int argc, char* argv[]) {
// serialize to sequence of bytes
opm_serialize_stream(ctx);
#if 0
// generate simpel stream
opm_dump_simple(ctx);
#else
// coplress the stream
opm_compress(ctx);
// concatenate streams
opm_concat_streams(ctx);
#endif
// dump raw data
opm_dump_events(ctx);
@ -821,6 +838,9 @@ int main(int argc, char* argv[]) {
// dump OPM file
opm_write_file(ctx);
// write statistics
// TODO
printf("done\n");
return 0;

View file

@ -88,6 +88,7 @@
<SDLCheck>true</SDLCheck>
<PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<ConformanceMode>true</ConformanceMode>
<RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
@ -102,6 +103,7 @@
<SDLCheck>true</SDLCheck>
<PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<ConformanceMode>true</ConformanceMode>
<RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
@ -140,10 +142,12 @@
</ItemDefinitionGroup>
<ItemGroup>
<ClCompile Include="cmdline.cpp" />
<ClCompile Include="compressor.cpp" />
<ClCompile Include="vgm2opl_next.cpp" />
</ItemGroup>
<ItemGroup>
<ClInclude Include="cmdline.h" />
<ClInclude Include="compressor.h" />
<ClInclude Include="opmctx.h" />
<ClInclude Include="opmfile.h" />
<ClInclude Include="vgm.h" />

View file

@ -21,6 +21,9 @@
<ClCompile Include="cmdline.cpp">
<Filter>Исходные файлы</Filter>
</ClCompile>
<ClCompile Include="compressor.cpp">
<Filter>Исходные файлы</Filter>
</ClCompile>
</ItemGroup>
<ItemGroup>
<ClInclude Include="vgm.h">
@ -35,5 +38,8 @@
<ClInclude Include="cmdline.h">
<Filter>Файлы заголовков</Filter>
</ClInclude>
<ClInclude Include="compressor.h">
<Filter>Файлы заголовков</Filter>
</ClInclude>
</ItemGroup>
</Project>