otomatalabs/moe-bius
7
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

port compression fixes from ESFM branch

Browse source
This commit is contained in:
wbcbz7 2024-06-06 03:36:21 +07:00
parent 6fca400cdd
commit 173bb1b11b
3 changed files with 111 additions and 30 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

43
vgm2opl_next/vgm2opl_next/compressor.cpp
View file

@ -36,7 +36,7 @@ static match_info_t test_match(
if (dst->flags & OPM_CHAN_BACKREF) {
// do recursive match
auto nested_match = test_match(srcdata, window, srcpos, dstpos - dst->distance_frames, depth - 1, dst->frames_to_play);
if (nested_match.logic_frames != dst->frames_to_play) break; else {
if ((dst->frames_to_play == 0) || (nested_match.logic_frames != dst->frames_to_play)) break; else {
// add this backref to match count
dstpos++;
srcpos += nested_match.logic_frames;
@ -46,12 +46,15 @@ static match_info_t test_match(
}
else {
// no backref, scan current frame
if ((src->rawdata != dst->rawdata) || (src->frame_dist != dst->frame_dist) ||
(rtn.distance_bytes + src->rawdata.size() >= MAX_BACKREF_DISTANCE_BYTES))
if ((src->rawdata != dst->rawdata) || (src->frame_dist != dst->frame_dist)) {
break;
}
else if (rtn.distance_bytes + src->rawdata.size() >= MAX_BACKREF_DISTANCE_BYTES)
{
// match failure or too long - enough for us :)
break;
} else {
}
else {
// match! increment everything
srcpos++; dstpos++;
rtn.logic_frames++;
@ -62,14 +65,15 @@ static match_info_t test_match(
} while (--max_length != 0);
// validate
if ((rtn.distance_bytes > MAX_BACKREF_DISTANCE_BYTES) ||
(rtn.logic_frames > MAX_BACKREF_DISTANCE_FRAMES)) return { 0 };
if ((rtn.distance_bytes >= MAX_BACKREF_DISTANCE_BYTES) ||
(rtn.logic_frames >= MAX_BACKREF_DISTANCE_FRAMES))
return { 0 };
return rtn;
}
// find match
static match_info_t find_match(opm_convert_context_t *ctx,
static match_info_t find_match(opm_convert_context_t* ctx,
std::vector<opm_channel_record_t>& srcdata, std::vector<opm_channel_record_t>& dstdata,
int current_pos, int max_lookback, int min_backref_length
) {
@ -83,6 +87,10 @@ static match_info_t find_match(opm_convert_context_t *ctx,
int srcpos = current_pos;
int dstpos = pos;
// check if max lookback in bytes is not violated
if ((srcdata[srcpos].byte_stamp - dstdata[dstpos].byte_stamp) > MAX_BACKREF_DISTANCE_BYTES)
continue;
// test current match
auto match = test_match(srcdata, dstdata, srcpos, dstpos, ctx->flags.max_stack_depth, MAX_BACKREF_LENGTH_FRAMES);
@ -92,6 +100,7 @@ static match_info_t find_match(opm_convert_context_t *ctx,
ret.distance_frames = dstdata.size() - pos;
ret.logic_frames = match.logic_frames;
ret.total_frames = match.total_frames;
ret.distance_bytes = match.distance_bytes;
return ret;
}
}
@ -107,14 +116,17 @@ uint32_t opm_compress_channel(opm_convert_context_t* ctx, int ch, int min_backre
// find match
auto match = find_match(ctx, src_ch, ctx->opmpacked[ch], pos, MAX_BACKREF_DISTANCE_FRAMES, min_backref_length);
if (match.logic_frames == 0) {
// copy one frame
total_bytes += src_ch[pos].rawdata.size();
// copy one frame, recalculate byte stamp!
opm_channel_record_t rec = src_ch[pos];
rec.byte_stamp = total_bytes;
ctx->opmpacked[ch].push_back(src_ch[pos]);
total_bytes += src_ch[pos].rawdata.size();
pos++;
}
else {
// set back reference - copy main parameters from original
opm_channel_record_t rec = src_ch[pos];
rec.byte_stamp = total_bytes;
rec.flags |= OPM_CHAN_BACKREF;
rec.distance_frames = match.distance_frames;
rec.frames_to_play = match.logic_frames;
@ -152,7 +164,7 @@ void opm_compress(opm_convert_context_t* ctx) {
int ch = 0;
for (auto& src_ch : ctx->opmrecords) {
if (ctx->flags.verbosity >= 2) {
printf("\n", ch);
printf("\n");
}
backref_bruteforce_t cur_backref_len;
cur_backref_len.min = backref_len.min;
@ -168,19 +180,22 @@ void opm_compress(opm_convert_context_t* ctx) {
}
}
// recompress if current != best
if (sz-1 != cur_backref_len.best) opm_compress_channel(ctx, ch, cur_backref_len.best);
// resolve back references
if (sz - 1 != cur_backref_len.best) opm_compress_channel(ctx, ch, cur_backref_len.best);
// calculate byte offsets for each frame
std::vector<uint32_t> ch_bytepos;
uint32_t bytepos_cur = 0;
for (int f = 0; f < ctx->opmpacked[ch].size(); f++) {
ch_bytepos.push_back(bytepos_cur);
bytepos_cur += ctx->opmpacked[ch][f].rawdata.size();
}
// resolve back references
for (int f = 0; f < ctx->opmpacked[ch].size(); f++) {
// fixup back reference (if any)
if (ctx->opmpacked[ch][f].flags & OPM_CHAN_BACKREF) {
uint32_t backref_dist = bytepos_cur - ch_bytepos[f - ctx->opmpacked[ch][f].distance_frames];
uint32_t backref_dist = ch_bytepos[f] - ch_bytepos[f - ctx->opmpacked[ch][f].distance_frames];
ctx->opmpacked[ch][f].rawdata[0] = OPM_STREAM_BACKREF | (backref_dist >> 8);
ctx->opmpacked[ch][f].rawdata[1] = (backref_dist & 0xFF);
}
bytepos_cur += ctx->opmpacked[ch][f].rawdata.size();
}
ch++;
#ifndef ULTRA_DEBUG

2
vgm2opl_next/vgm2opl_next/opmctx.h
View file

@ -98,6 +98,8 @@ struct opm_channel_record_t {
std::vector<opm_frame_record> records;
// compression data
int distance_frames, distance_bytes, frames_to_play, frames_to_play_total;
// byte stamp
uint32_t byte_stamp;
};

96
vgm2opl_next/vgm2opl_next/vgm2opl_next.cpp
View file

@ -144,10 +144,10 @@ int chanParamsToChan[16] = {
// ------------------------
// 2nd step - extract register writes from VGM, requantize to new frames
int opm_requantize(opm_convert_context_t* ctx) {
ctx->oplchan.resize(1 + 9); // TODO: FIXME for OPL3 support!!
ctx->oplchan.resize(1 + 18);
// per-channel structures
std::vector<opm_channel_rawdata_t> oplchans(1 + 9); // TODO
std::vector<opm_channel_rawdata_t> oplchans(1 + 18); // TODO
uint32_t currentFrame = 0; uint32_t delay_rate, delay_acc = 0;
auto it = ctx->vgm.vgmfile.begin() + ctx->vgm.start; bool loopFrame = false;
@ -522,7 +522,7 @@ int opm_group_channel_stream(opm_convert_context_t* ctx, int ch) {
// 3rd step - group events by registers written
// key changes are acting as "fence" (the entire stream is flushed in this case)
int opm_group_registers(opm_convert_context_t* ctx) {
ctx->opmrecords.resize(9 + 1);
ctx->opmrecords.resize(18 + 1);
opm_group_control_stream(ctx);
for (int ch = 0; ch < 9; ch++) {
@ -561,6 +561,9 @@ int opm_serialize_control_stream(opm_convert_context_t* ctx) {
for (auto& s : ctx->opmrecords[0]) {
s.rawdata.clear();
// set loop point
if (s.flags & OPM_CHAN_LOOP_POINT) s.rawdata.push_back(OPM_STREAM_LOOP);
// set delay
if (s.frame_dist != old_delay) {
old_delay = s.frame_dist;
@ -586,9 +589,13 @@ int opm_serialize_control_stream(opm_convert_context_t* ctx) {
int opm_serialize_channel_stream(opm_convert_context_t* ctx, int ch) {
int old_delay = -1;
uint32_t byte_stamp = 0;
for (auto& s : ctx->opmrecords[ch]) {
s.rawdata.clear();
// set loop point
if (s.flags & OPM_CHAN_LOOP_POINT) s.rawdata.push_back(OPM_STREAM_LOOP);
// set delay
if (s.frame_dist != old_delay) {
old_delay = s.frame_dist;
@ -669,6 +676,10 @@ int opm_serialize_channel_stream(opm_convert_context_t* ctx, int ch) {
}
} else
s.rawdata.push_back(OPM_STREAM_END_FRAME);
// save byte stamp
s.byte_stamp = byte_stamp;
byte_stamp += s.rawdata.size();
}
return 0;
}
@ -700,29 +711,59 @@ int opm_concat_streams(opm_convert_context_t* ctx) {
}
int opm_dump_backref(opm_convert_context_t* ctx, FILE* f, int ch, int pos, int frames, int current_frame) {
if (ctx->opmpacked[ch].size() > 0) do {
auto& a = ctx->opmpacked[ch][pos];
if (a.flags & OPM_CHAN_BACKREF) {
current_frame = opm_dump_backref(ctx, f, ch, pos - a.distance_frames, a.frames_to_play, current_frame);
}
else {
fprintf(f, "frame %07d, dist %d: data ", current_frame, a.frame_dist);
for (auto& d : a.rawdata) fprintf(f, "%02X ", d);
fprintf(f, "\n");
current_frame += a.frame_dist;
}
pos++;
} while ((--frames != 0) && (pos < ctx->opmpacked[ch].size()));
return current_frame;
}
// ---------------------------
// dump events for each channel
int opm_dump_events(opm_convert_context_t* ctx) {
FILE* f = fopen(ctx->logname.c_str(), "w");
fprintf(f, "total %d frames\n", ctx->total_frames);
for (int ch = 0; ch < (1 + 9); ch++) {
fprintf(f, "---channel %d\n", ch-1);
for (int ch = 0; ch < (ctx->opmrecords.size()); ch++) {
fprintf(f, "---channel %d\n", ch - 1);
for (auto& a : ctx->opmrecords[ch]) {
fprintf(f, "frame %07d, dist %d: data ", a.frame, a.frame_dist);
for (auto& d : a.rawdata) fprintf(f, "%02X ", d);
fprintf(f,"\n");
fprintf(f, "\n");
}
}
fclose(f);
f = fopen((ctx->logname+".packed.log").c_str(), "w");
fprintf(f, "total %d frames\n", ctx->total_frames);
for (int ch = 0; ch < (1 + 9); ch++) {
fprintf(f, "---channel %d\n", ch - 1);
for (auto& a : ctx->opmpacked[ch]) {
fprintf(f, "frame %07d, dist %d: data ", a.frame, a.frame_dist);
for (auto& d : a.rawdata) fprintf(f, "%02X ", d);
fprintf(f, "\n");
f = fopen((ctx->logname + ".packed.log").c_str(), "w");
if (ctx->flags.compress_level > 0) {
fprintf(f, "total %d frames\n", ctx->total_frames);
for (int ch = 0; ch < (ctx->opmpacked.size()); ch++) {
fprintf(f, "---channel %d\n", ch - 1);
if (ctx->opmpacked[ch].size() > 0) for (auto& a : ctx->opmpacked[ch]) {
fprintf(f, "frame %07d, dist %d: data ", a.frame, a.frame_dist);
for (auto& d : a.rawdata) fprintf(f, "%02X ", d);
fprintf(f, "\n");
}
}
}
fclose(f);
// dump decompressed statistics
f = fopen((ctx->logname + ".unpacked.log").c_str(), "w");
if (ctx->flags.compress_level > 0) {
fprintf(f, "total %d frames\n", ctx->total_frames);
for (int ch = 0; ch < (ctx->opmpacked.size()); ch++) {
fprintf(f, "---channel %d\n", ch - 1);
opm_dump_backref(ctx, f, ch, 0, -1, 0);
}
}
fclose(f);
@ -742,14 +783,14 @@ const cmdline_t cmdparams[] = {
int main(int argc, char* argv[]) {
// clear compression context!
ctx->flags.compress_level = 1;
ctx->flags.max_stack_depth = 2;
ctx->flags.max_stack_depth = 1;
ctx->flags.verbosity = 1;
if (argc < 2) {
printf("usage: vgm2opm input.vgm [-cx] [-dx] [-vx]\n");
printf("-c[x]: enable compression, [x] - mode (default - 1):\n");
printf(" 0 - disabled, 1 - fixed min backref, 2 - bruteforce best backref\n");
printf("-d[x]: maximum back reference stack depth (default - 2)\n");
printf("-d[x]: maximum back reference stack depth (default - 1)\n");
printf("-v[x]: verbosity level (default - 1)\n");
return 1;
}
@ -770,6 +811,7 @@ int main(int argc, char* argv[]) {
ctx->opmfile.filename = outfile_str;
ctx->logname = logfile_str;
#if 0
std::ifstream infile(infile_str, std::ios::in | std::ios::binary);
infile.unsetf(std::ios::skipws);
@ -781,6 +823,28 @@ int main(int argc, char* argv[]) {
// read whole file
ctx->vgm.vgmfile.reserve(fsize);
ctx->vgm.vgmfile.insert(ctx->vgm.vgmfile.begin(), std::istream_iterator<uint8_t>(infile), std::istream_iterator<uint8_t>());
#else
// good old stdio :)
FILE* infile = fopen(infile_str.c_str(), "rb");
if (infile == nullptr) {
printf("unable to open %s!\n", infile_str.c_str());
return 1;
}
_fseeki64(infile, 0, SEEK_END);
uint64_t fsize = _ftelli64(infile);
_fseeki64(infile, 0, SEEK_SET);
// read whole file
ctx->vgm.vgmfile.resize(fsize);
if (fread(ctx->vgm.vgmfile.data(), sizeof(uint8_t), fsize, infile) != fsize) {
printf("unable to read VGM file!\n");
return 1;
};
// done
fclose(infile);
#endif
// get header
ctx->vgm.header = reinterpret_cast<VGMHeader*>(ctx->vgm.vgmfile.data());