add big endian and long ptr options to cmd stream

the format also changes!

papers/export-tech.md

@@ -19,12 +19,17 @@ Furnace Command Stream, split version.
 size | description
 -----|------------------------------------
   4  | "FCS\0" format magic
-  4  | channel count
+  2  | channel count
- 4?? | pointers to channel data
+  1  | flags
+     | - bit 1: big-endian addresses
+     | - bit 0: pointer size (off: short; on: long)
+  1  | reserved
  1?? | preset delays
      | - 16 values
  1?? | speed dial commands
      | - 16 values
+ ??? | pointers to channel data
+     | - pointers are short (2-byte) or long (4-byte), set in flags
  ??? | channel data
 ```
 

src/engine/cmdStream.cpp

@@ -173,7 +173,7 @@ bool DivCSPlayer::tick() {
           command=stream.readC();
           break;
         case 0xf8: {
-          unsigned int callAddr=(unsigned short)stream.readS();
+          unsigned int callAddr=bigEndian?((unsigned short)stream.readS_BE()):((unsigned short)stream.readS());
           chan[i].readPos=stream.tell();
           if (!chan[i].doCall(callAddr)) {
             logE("%d: (call) stack error!",i);
@@ -183,7 +183,7 @@ bool DivCSPlayer::tick() {
           break;
         }
         case 0xf5: {
-          unsigned int callAddr=stream.readI();
+          unsigned int callAddr=bigEndian?stream.readI_BE():stream.readI();
           chan[i].readPos=stream.tell();
           if (!chan[i].doCall(callAddr)) {
             logE("%d: (calli) stack error!",i);
@@ -207,7 +207,7 @@ bool DivCSPlayer::tick() {
           mustTell=false;
           break;
         case 0xfa:
-          chan[i].readPos=stream.readI();
+          chan[i].readPos=bigEndian?stream.readI_BE():stream.readI();
           mustTell=false;
           break;
         case 0xfb:
@@ -215,7 +215,7 @@ bool DivCSPlayer::tick() {
           stream.readI();
           break;
         case 0xfc:
-          chan[i].waitTicks=(unsigned short)stream.readS();
+          chan[i].waitTicks=(unsigned short)(bigEndian?stream.readS_BE():stream.readS());
           chan[i].lastWaitLen=chan[i].waitTicks;
           break;
         case 0xfd:
@@ -268,11 +268,11 @@ bool DivCSPlayer::tick() {
             arg0=(arg0&0x80)?1:0;
             break;
           case DIV_CMD_HINT_VOL_SLIDE:
-            arg0=(short)stream.readS();
+            arg0=(short)(bigEndian?stream.readS_BE():stream.readS());
             break;
           case DIV_CMD_HINT_VOL_SLIDE_TARGET:
-            arg0=(short)stream.readS();
+            arg0=(short)(bigEndian?stream.readS_BE():stream.readS());
-            arg1=(short)stream.readS();
+            arg1=(short)(bigEndian?stream.readS_BE():stream.readS());
             break;
           case DIV_CMD_HINT_LEGATO:
             arg0=(unsigned char)stream.readC();
@@ -479,16 +479,16 @@ bool DivCSPlayer::tick() {
           case DIV_CMD_LYNX_LFSR_LOAD:
           case DIV_CMD_QSOUND_ECHO_DELAY:
           case DIV_CMD_ES5506_ENVELOPE_COUNT:
-            arg0=(unsigned short)stream.readS();
+            arg0=(unsigned short)(bigEndian?stream.readS_BE():stream.readS());
             break;
           // TWO SHORT COMMANDS
           case DIV_CMD_ES5506_FILTER_K1:
           case DIV_CMD_ES5506_FILTER_K2:
-            arg0=(unsigned short)stream.readS();
+            arg0=(unsigned short)(bigEndian?stream.readS_BE():stream.readS());
-            arg1=(unsigned short)stream.readS();
+            arg1=(unsigned short)(bigEndian?stream.readS_BE():stream.readS());
             break;
           case DIV_CMD_FM_FIXFREQ:
-            arg0=(unsigned short)stream.readS();
+            arg0=(unsigned short)(bigEndian?stream.readS_BE():stream.readS());
             arg1=arg0&0x7ff;
             arg0>>=12;
             break;
@@ -498,7 +498,7 @@ bool DivCSPlayer::tick() {
             arg0=(arg0&8)?1:0;
             break;
           case DIV_CMD_SAMPLE_POS:
-            arg0=(unsigned int)stream.readI();
+            arg0=(unsigned int)(bigEndian?stream.readI_BE():stream.readI());
             break;
         }
 
@@ -645,8 +645,21 @@ bool DivCSPlayer::init() {
 
   if (memcmp(magic,"FCS",4)!=0) return false;
 
-  fileChans=stream.readI();
+  fileChans=(unsigned short)stream.readS();
+  unsigned char flags=stream.readC();
+  stream.readC(); // reserved
 
+  longPointers=flags&1;
+  bigEndian=flags&2;
+
+  if (bigEndian) fileChans=(((fileChans&0xff00)>>8)|((fileChans&0xff)<<8));
+
+  fastDelaysOff=stream.tell();
+  stream.read(fastDelays,16);
+  fastCmdsOff=stream.tell();
+  stream.read(fastCmds,16);
+
+  if (longPointers) {
     for (unsigned int i=0; i<fileChans; i++) {
       if (i>=DIV_MAX_CHANS) {
         stream.readI();
@@ -656,14 +669,31 @@ bool DivCSPlayer::init() {
         stream.readI();
         continue;
       }
+      if (bigEndian) {
+        chan[i].startPos=stream.readI_BE();
+      } else {
         chan[i].startPos=stream.readI();
+      }
       chan[i].readPos=chan[i].startPos;
     }
-
+  } else {
-  fastDelaysOff=stream.tell();
+    for (unsigned int i=0; i<fileChans; i++) {
-  stream.read(fastDelays,16);
+      if (i>=DIV_MAX_CHANS) {
-  fastCmdsOff=stream.tell();
+        stream.readS();
-  stream.read(fastCmds,16);
+        continue;
+      }
+      if ((int)i>=e->getTotalChannelCount()) {
+        stream.readS();
+        continue;
+      }
+      if (bigEndian) {
+        chan[i].startPos=stream.readS_BE();
+      } else {
+        chan[i].startPos=stream.readS();
+      }
+      chan[i].readPos=chan[i].startPos;
+    }
+  }
 
   // initialize state
   for (int i=0; i<e->getTotalChannelCount(); i++) {

src/engine/cmdStream.h

@@ -87,6 +87,8 @@ class DivCSPlayer {
   unsigned char arpSpeed;
   unsigned int fileChans;
   unsigned int curTick, fastDelaysOff, fastCmdsOff, deltaCyclePos;
+  bool longPointers;
+  bool bigEndian;
 
   short vibTable[64];
   public:
@@ -117,6 +119,21 @@ struct DivCSProgress {
     total(0) {}
 };
 
+struct DivCSOptions {
+  bool longPointers;
+  bool bigEndian;
+  bool noCmdCallOpt;
+  bool noDelayCondense;
+  bool noSubBlock;
+
+  DivCSOptions():
+    longPointers(false),
+    bigEndian(false),
+    noCmdCallOpt(false),
+    noDelayCondense(false),
+    noSubBlock(false) {}
+};
+
 // command stream utilities
 namespace DivCS {
   int getCmdLength(unsigned char ext);

src/engine/cmdStreamOps.cpp

@@ -273,7 +273,7 @@ int DivCS::getInsLength(unsigned char ins, unsigned char ext, unsigned char* spe
   return 1;
 }
 
-void writeCommandValues(SafeWriter* w, const DivCommand& c) {
+void writeCommandValues(SafeWriter* w, const DivCommand& c, bool bigEndian) {
   switch (c.cmd) {
     case DIV_CMD_NOTE_ON:
       if (c.value==DIV_NOTE_NULL) {
@@ -342,11 +342,20 @@ void writeCommandValues(SafeWriter* w, const DivCommand& c) {
       w->writeC((c.value?0x80:0)|(c.value2?0x40:0));
       break;
     case DIV_CMD_HINT_VOL_SLIDE:
+      if (bigEndian) {
+        w->writeS_BE(c.value);
+      } else {
         w->writeS(c.value);
+      }
       break;
     case DIV_CMD_HINT_VOL_SLIDE_TARGET:
+      if (bigEndian) {
+        w->writeS_BE(c.value);
+        w->writeS_BE(c.value2);
+      } else {
         w->writeS(c.value);
         w->writeS(c.value2);
+      }
       break;
     case DIV_CMD_SAMPLE_MODE:
     case DIV_CMD_SAMPLE_FREQ:
@@ -542,21 +551,38 @@ void writeCommandValues(SafeWriter* w, const DivCommand& c) {
     case DIV_CMD_LYNX_LFSR_LOAD:
     case DIV_CMD_QSOUND_ECHO_DELAY:
     case DIV_CMD_ES5506_ENVELOPE_COUNT:
+      if (bigEndian) {
+        w->writeS_BE(c.value);
+      } else {
         w->writeS(c.value);
+      }
       break;
     case DIV_CMD_ES5506_FILTER_K1:
     case DIV_CMD_ES5506_FILTER_K2:
+      if (bigEndian) {
+        w->writeS_BE(c.value);
+        w->writeS_BE(c.value2);
+      } else {
         w->writeS(c.value);
         w->writeS(c.value2);
+      }
       break;
     case DIV_CMD_FM_FIXFREQ:
+      if (bigEndian) {
+        w->writeS_BE((c.value<<12)|(c.value2&0x7ff));
+      } else {
         w->writeS((c.value<<12)|(c.value2&0x7ff));
+      }
       break;
     case DIV_CMD_NES_SWEEP:
       w->writeC((c.value?8:0)|(c.value2&0x77));
       break;
     case DIV_CMD_SAMPLE_POS:
+      if (bigEndian) {
+        w->writeI_BE(c.value);
+      } else {
         w->writeI(c.value);
+      }
       break;
     default:
       logW("unimplemented command %s!",cmdName[c.cmd]);
@@ -611,7 +637,7 @@ void reloc8(unsigned char* buf, size_t len, unsigned int sourceAddr, unsigned in
   }
 }
 
-void reloc(unsigned char* buf, size_t len, unsigned int sourceAddr, unsigned int destAddr, unsigned char* speedDial) {
+void reloc(unsigned char* buf, size_t len, unsigned int sourceAddr, unsigned int destAddr, unsigned char* speedDial, bool bigEndian) {
   unsigned int delta=destAddr-sourceAddr;
   for (size_t i=0; i<len;) {
     int insLen=getInsLength(buf[i],_EXT(buf,i,len),speedDial);
@@ -624,17 +650,29 @@ void reloc(unsigned char* buf, size_t len, unsigned int sourceAddr, unsigned int
       case 0xfa: { // jmp
         unsigned int addr=buf[i+1]|(buf[i+2]<<8)|(buf[i+3]<<16)|(buf[i+4]<<24);
         addr+=delta;
+        if (bigEndian) {
+          buf[i+1]=(addr>>24)&0xff;
+          buf[i+2]=(addr>>16)&0xff;
+          buf[i+3]=(addr>>8)&0xff;
+          buf[i+4]=addr&0xff;
+        } else {
           buf[i+1]=addr&0xff;
           buf[i+2]=(addr>>8)&0xff;
           buf[i+3]=(addr>>16)&0xff;
           buf[i+4]=(addr>>24)&0xff;
+        }
         break;
       }
       case 0xf8: { // call
         unsigned short addr=buf[i+1]|(buf[i+2]<<8);
         addr+=delta;
+        if (bigEndian) {
+          buf[i+1]=(addr>>8)&0xff;
+          buf[i+2]=addr&0xff;
+        } else {
           buf[i+1]=addr&0xff;
           buf[i+2]=(addr>>8)&0xff;
+        }
         break;
       }
     }
@@ -1123,7 +1161,7 @@ SafeWriter* packStream(SafeWriter* s, unsigned char* speedDial) {
   return s;
 }
 
-SafeWriter* DivEngine::saveCommand(DivCSProgress* progress, unsigned int disablePasses) {
+SafeWriter* DivEngine::saveCommand(DivCSProgress* progress, DivCSOptions options) {
   stop();
   repeatPattern=false;
   shallStop=false;
@@ -1167,16 +1205,24 @@ SafeWriter* DivEngine::saveCommand(DivCSProgress* progress, unsigned int disable
 
   // write header
   w->write("FCS",4);
-  w->writeI(chans);
+  w->writeS(chans);
+  // flags
+  w->writeC((options.longPointers?1:0)|(options.bigEndian?2:0));
+  // reserved
+  w->writeC(0);
+  // preset delays and speed dial
+  for (int i=0; i<32; i++) {
+    w->writeC(0);
+  }
   // offsets
   for (int i=0; i<chans; i++) {
     chanStream[i]=new SafeWriter;
     chanStream[i]->init();
+    if (options.longPointers) {
       w->writeI(0);
+    } else {
+      w->writeS(0);
     }
-  // preset delays and speed dial
-  for (int i=0; i<32; i++) {
-    w->writeC(0);
   }
 
   // play the song ourselves
@@ -1252,7 +1298,7 @@ SafeWriter* DivEngine::saveCommand(DivCSProgress* progress, unsigned int disable
           break;
         default:
           cmdPopularity[i.cmd]++;
-          writeCommandValues(chanStream[i.chan],i);
+          writeCommandValues(chanStream[i.chan],i,options.bigEndian);
           break;
       }
     }
@@ -1316,7 +1362,7 @@ SafeWriter* DivEngine::saveCommand(DivCSProgress* progress, unsigned int disable
   BUSY_END;
 
   // PASS 1: optimize command calls
-  if (!(disablePasses&1)) {
+  if (!options.noCmdCallOpt) {
     // calculate command usage
     int sortCand=-1;
     int sortPos=0;
@@ -1361,7 +1407,7 @@ SafeWriter* DivEngine::saveCommand(DivCSProgress* progress, unsigned int disable
   }
 
   // PASS 2: condense delays
-  if (!(disablePasses&2)) {
+  if (!options.noDelayCondense) {
     // calculate delay usage
     for (int h=0; h<chans; h++) {
       unsigned char* buf=chanStream[h]->getFinalBuf();
@@ -1471,7 +1517,7 @@ SafeWriter* DivEngine::saveCommand(DivCSProgress* progress, unsigned int disable
   }
 
   // PASS 5: find sub-blocks and isolate them
-  if (!(disablePasses&4)) {
+  if (!options.noSubBlock) {
     std::vector<SafeWriter*> subBlocks;
     size_t beforeSize=globalStream->size();
     
@@ -1562,16 +1608,29 @@ SafeWriter* DivEngine::saveCommand(DivCSProgress* progress, unsigned int disable
     }
   }
 
-  // write results
+  // write results (convert addresses to big-endian if necessary)
-  reloc(globalStream->getFinalBuf(),globalStream->size(),0,w->tell(),sortedCmd);
+  reloc(globalStream->getFinalBuf(),globalStream->size(),0,w->tell(),sortedCmd,options.bigEndian);
   w->write(globalStream->getFinalBuf(),globalStream->size());
 
-  w->seek(8,SEEK_SET);
+  w->seek(40,SEEK_SET);
   for (int i=0; i<chans; i++) {
+    if (options.longPointers) {
+      if (options.bigEndian) {
+        w->writeI_BE(chanStreamOff[i]);
+      } else {
         w->writeI(chanStreamOff[i]);
       }
+    } else {
+      if (options.bigEndian) {
+        w->writeS_BE(chanStreamOff[i]);
+      } else {
+        w->writeS(chanStreamOff[i]);
+      }
+    }
+  }
 
   logD("delay popularity:");
+  w->seek(8,SEEK_SET);
   for (int i=0; i<16; i++) {
     w->writeC(sortedDelay[i]);
     if (sortedDelayPopularity[i]) logD("- %d: %d",sortedDelay[i],sortedDelayPopularity[i]);

src/engine/engine.h

@@ -730,7 +730,7 @@ class DivEngine {
     // dump to TIunA.
     SafeWriter* saveTiuna(const bool* sysToExport, const char* baseLabel, int firstBankSize, int otherBankSize);
     // dump command stream.
-    SafeWriter* saveCommand(DivCSProgress* progress=NULL, unsigned int disablePasses=0);
+    SafeWriter* saveCommand(DivCSProgress* progress=NULL, DivCSOptions options=DivCSOptions());
     // export to text
     SafeWriter* saveText(bool separatePatterns=true);
     // export to an audio file

src/gui/exportOptions.cpp

@@ -381,22 +381,12 @@ void FurnaceGUI::drawExportText(bool onWindow) {
 }
 
 void FurnaceGUI::commandExportOptions() {
-  bool noCmdCallOpt=(csExportDisablePass&1);
+  ImGui::Checkbox(_("Long pointers (use for 64K+ size streams)"),&csExportOptions.longPointers);
-  bool noDelayCondense=(csExportDisablePass&2);
+  ImGui::Checkbox(_("Big endian mode"),&csExportOptions.bigEndian);
-  bool noSubBlock=(csExportDisablePass&4);
+  ImGui::Separator();
-
+  ImGui::Checkbox(_("Don't optimize command calls"),&csExportOptions.noCmdCallOpt);
-  if (ImGui::Checkbox(_("Don't optimize command calls"),&noCmdCallOpt)) {
+  ImGui::Checkbox(_("Don't condense delays"),&csExportOptions.noDelayCondense);
-    csExportDisablePass&=~1;
+  ImGui::Checkbox(_("Don't perform sub-block search"),&csExportOptions.noSubBlock);
-    csExportDisablePass|=noCmdCallOpt?1:0;
-  }
-  if (ImGui::Checkbox(_("Don't condense delays"),&noDelayCondense)) {
-    csExportDisablePass&=~2;
-    csExportDisablePass|=noDelayCondense?2:0;
-  }
-  if (ImGui::Checkbox(_("Don't perform sub-block search"),&noSubBlock)) {
-    csExportDisablePass&=~4;
-    csExportDisablePass|=noSubBlock?4:0;
-  }
 }
 
 void FurnaceGUI::drawExportCommand(bool onWindow) {

src/gui/gui.cpp

@@ -2653,7 +2653,7 @@ void FurnaceGUI::exportCmdStream(bool target, String path) {
   csExportTarget=target;
   csExportDone=false;
   csExportThread=new std::thread([this]() {
-    SafeWriter* w=e->saveCommand(&csProgress,csExportDisablePass);
+    SafeWriter* w=e->saveCommand(&csProgress,csExportOptions);
     csExportResult=w;
     csExportDone=true;
   });
@@ -8951,7 +8951,6 @@ FurnaceGUI::FurnaceGUI():
   csExportDone(false),
   dmfExportVersion(0),
   curExportType(GUI_EXPORT_NONE),
-  csExportDisablePass(0),
   romTarget(DIV_ROM_ABSTRACT),
   romMultiFile(false),
   romExportSave(false),

src/gui/gui.h

@@ -2765,7 +2765,7 @@ class FurnaceGUI {
   DivAudioExportOptions audioExportOptions;
   int dmfExportVersion;
   FurnaceGUIExportTypes curExportType;
-  unsigned int csExportDisablePass;
+  DivCSOptions csExportOptions;
   DivCSProgress csProgress;
 
   // ROM export specific

src/main.cpp

@@ -90,7 +90,7 @@ String romOutName;
 String txtOutName;
 int benchMode=0;
 int subsong=-1;
-int cmdDisableOpt=0;
+DivCSOptions csExportOptions;
 DivAudioExportOptions exportOptions;
 DivConfig romExportConfig;
 
@@ -442,17 +442,6 @@ TAParamResult pCmdOut(String val) {
   return TA_PARAM_SUCCESS;
 }
 
-TAParamResult pCmdOpt(String val) {
-  try {
-    int v=std::stoi(val);
-    cmdDisableOpt=v;
-  } catch (std::exception& e) {
-    logE("command stream export optimization disable bitmask shall be a number.");
-    return TA_PARAM_ERROR;
-  }
-  return TA_PARAM_SUCCESS;
-}
-
 TAParamResult pROMOut(String val) {
   romOutName=val;
   e.setAudio(DIV_AUDIO_DUMMY);
@@ -494,7 +483,6 @@ void initParams() {
   params.push_back(TAParam("O","vgmout",true,pVGMOut,"<filename>","output .vgm data"));
   params.push_back(TAParam("D","direct",false,pDirect,"","set VGM export direct stream mode"));
   params.push_back(TAParam("C","cmdout",true,pCmdOut,"<filename>","output command stream"));
-  params.push_back(TAParam("","cmdopt",true,pCmdOpt,"<bitmask>","disable command stream optimization passes (+1 command, +2 delay, +4 sub-block)"));
   params.push_back(TAParam("r","romout",true,pROMOut,"<filename|path>","export ROM file, or path for multi-file export"));
   params.push_back(TAParam("R","romconf",true,pROMConf,"<key>=<value>","set configuration parameter for ROM export"));
   params.push_back(TAParam("t","txtout",true,pTxtOut,"<filename>","export as text file"));
@@ -894,7 +882,7 @@ int main(int argc, char** argv) {
 
   if (outputMode) {
     if (cmdOutName!="") {
-      SafeWriter* w=e.saveCommand(NULL,cmdDisableOpt);
+      SafeWriter* w=e.saveCommand(NULL);
       if (w!=NULL) {
         FILE* f=ps_fopen(cmdOutName.c_str(),"wb");
         if (f!=NULL) {