fix: make stage4 files sort-private, enable sub-buffer stage4 sorts

check/fixes.frm

@@ -2162,13 +2162,123 @@ Print;
 .end
 # Only for 64-bit systems. Otherwise "Output term too large".
 #require wordsize == 4
-# For now it fails because
-#   "Currently Stage 4 sorts are not allowed for function arguments or $ variables."
-assert runtime_error?
-#assert succeeded?
-#assert result("test1") =~ expr("0")
-#assert result("test2") =~ expr("g(0)")
+assert warning?("StageSort in sub-buffer: recommend increasing setup sub-buffer sizes")
+assert result("test1") =~ expr("0")
+assert result("test2") =~ expr("g(0)")
 *--#] Issue211 : 
+*--#[ Issue215_1 :
+#-
+
+#: SubTermsInSmall 16
+#: SubLargePatches 4
+#: SubFilePatches 4
+
+Off statistics;
+Off threadstats;
+
+CFunction f;
+Symbol j,x;
+
+#define BLOWUP "1000"
+
+Local argument = f(x) + f(2*x);
+Argument f;
+	Identify x = sum_(j,1,`BLOWUP',x*j/`BLOWUP');
+EndArgument;
+Identify f(x?) = x;
+Identify x = 1;
+Print;
+.end
+assert warning?("StageSort in sub-buffer: recommend increasing setup sub-buffer sizes")
+assert result("argument") =~ expr("3003/2")
+*--#] Issue215_1 :
+*--#[ Issue215_2 :
+#-
+
+#: SubTermsInSmall 16
+#: SubLargePatches 4
+#: SubFilePatches 4
+
+Off statistics;
+Off threadstats;
+
+Symbol j,x;
+
+#define BLOWUP "1000"
+
+#$dol = x;
+Local dollar = 1;
+Inside $dol;
+	Identify x = sum_(j,1,`BLOWUP',x*j/`BLOWUP');
+EndInside;
+Multiply $dol;
+Identify x = 1;
+Print;
+ModuleOption local $dol;
+.end
+assert warning?("StageSort in sub-buffer: recommend increasing setup sub-buffer sizes")
+assert result("dollar") =~ expr("1001/2")
+*--#] Issue215_2 :
+*--#[ Issue215_3 :
+#-
+
+#: SubTermsInSmall 16
+#: SubLargePatches 4
+#: SubFilePatches 4
+
+Off statistics;
+Off threadstats;
+
+Symbol j,x;
+
+#define BLOWUP "1000"
+
+Local term = x;
+Term;
+	Identify x = sum_(j,1,`BLOWUP',x*j/`BLOWUP');
+EndTerm;
+Identify x = 1;
+Print;
+.end
+assert warning?("StageSort in sub-buffer: recommend increasing setup sub-buffer sizes")
+assert result("term") =~ expr("1001/2")
+*--#] Issue215_3 :
+*--#[ Issue215_4 :
+#-
+
+#: TermsInSmall 16
+#: LargePatches 4
+#: FilePatches 4
+
+#: SubTermsInSmall 16
+#: SubLargePatches 4
+#: SubFilePatches 4
+
+Off statistics;
+Off threadstats;
+
+CFunction f;
+Symbol i,j,x;
+
+#define N "500"
+#define BLOWUP "1000"
+
+Local nested = {`N'*(`N'+1)/2}
+	#do i = 1,`N'
+		- f(x^`i')
+	#enddo
+	;
+
+Argument f;
+	Identify x^i? = sum_(j,1,`BLOWUP',x^i*i/`BLOWUP');
+EndArgument;
+Identify f(x?) = x;
+Identify x^i? = 1;
+Print;
+.end
+assert warning?("StageSort in sub-buffer: recommend increasing setup sub-buffer sizes")
+assert result("nested") =~ expr("0")
+*--#] Issue215_4 :
 *--#[ Issue222 :
 * accessing #factdollar factors causes program termination
 Symbol x;

doc/manual/sorting.tex

@@ -254,10 +254,10 @@ \chapter{Sorting and statistics}
 the chapter on the setup \ref{setup}. This chapter gives also all default 
 values.
 
-There is one restriction on the sorting of function arguments and 
-\$-expressions: They are not allowed to go into the stage4 sorting. Any 
-such attempt will result in an error message and the suggestion to raise 
-the size of the buffers for this type of sorting.
+Sorting these `sub-buffers' may also run into a stage 4 sort when necessary.
+This usually implies that performance will be severely affected, hence the
+first time this happens during a program a warning will be printed, with the
+suggestion to increase the sizes of the sub-buffers.
 
 When \FORM\ is running in parallel mode (either \TFORM\ or \ParFORM) each worker 
 will need its own buffers. In \ParFORM\ in which the processors each control 

sources/checkpoint.c

@@ -1184,7 +1184,6 @@ static void print_R()
 	MesPrint("%d", AR.DeferFlag);
 	MesPrint("%d", AR.TePos);
 	MesPrint("%d", AR.sLevel);
-	MesPrint("%d", AR.Stage4Name);
 	MesPrint("%d", AR.GetOneFile);
 	MesPrint("%d", AR.PolyFun);
 	MesPrint("%d", AR.PolyFunInv);
@@ -2333,7 +2332,6 @@ int DoRecovery(int *moduletype)
 	R_SET(AR.DeferFlag, WORD);
 	R_SET(AR.TePos, WORD);
 	R_SET(AR.sLevel, WORD);
-	R_SET(AR.Stage4Name, WORD);
 	R_SET(AR.GetOneFile, WORD);
 	R_SET(AR.PolyFun, WORD);
 	R_SET(AR.PolyFunInv, WORD);
@@ -2957,7 +2955,6 @@ static int DoSnapshot(int moduletype)
 	S_WRITE_B(&AR.DeferFlag, sizeof(WORD));
 	S_WRITE_B(&AR.TePos, sizeof(WORD));
 	S_WRITE_B(&AR.sLevel, sizeof(WORD));
-	S_WRITE_B(&AR.Stage4Name, sizeof(WORD));
 	S_WRITE_B(&AR.GetOneFile, sizeof(WORD));
 	S_WRITE_B(&AR.PolyFun, sizeof(WORD));
 	S_WRITE_B(&AR.PolyFunInv, sizeof(WORD));

sources/proces.c

@@ -205,6 +205,7 @@ int Processor(void)
 #endif
 		AS.CollectOverFlag = 0;
 		AR.expchanged = 0;
+		AT.SubSortStage4Warning = 0;
 		if ( i == last ) LastExpression = 1;
 		else             LastExpression = 0;
 		if ( e->inmem ) {

sources/setfile.c

@@ -417,10 +417,10 @@ int AllocSetups(void)
 	SETUPPARAMETERS *sp;
 	LONG LargeSize, SmallSize, SmallEsize, TermsInSmall, IOsize;
 	int MaxPatches, MaxFpatches, error = 0, i, size;
-	UBYTE *s;
 #ifndef WITHPTHREADS
 	int j;
 #endif
+	UBYTE *s;
 	sp = GetSetupPar((UBYTE *)"threads");
 	if ( sp->value > 0 ) AM.totalnumberofthreads = sp->value+1;
 
@@ -588,19 +588,14 @@ int AllocSetups(void)
 	/* AM.S0->file.ziosize was already set to a (larger) value by AllocSort, here it is re-set. */
 #ifdef WITHZLIB
 	AM.S0->file.ziosize = IOsize;
+	AM.S0->FoStage4.ziosize = IOsize;
 #ifndef WITHPTHREADS
-	AR.FoStage4[0].ziosize = IOsize;
-	AR.FoStage4[1].ziosize = IOsize;
 	AT.S0 = AM.S0;
 #endif
 #else
 #ifndef WITHPTHREADS
 	AT.S0 = AM.S0;
 #endif
-#endif
-#ifndef WITHPTHREADS
-	AR.FoStage4[0].POsize   = ((IOsize+sizeof(WORD)-1)/sizeof(WORD))*sizeof(WORD);
-	AR.FoStage4[1].POsize   = ((IOsize+sizeof(WORD)-1)/sizeof(WORD))*sizeof(WORD);
 #endif
 	sp = GetSetupPar((UBYTE *)"subsmallsize");
 	AM.SSmallSize = sp->value;
@@ -998,6 +993,17 @@ SORTING *AllocSort(LONG inLargeSize, LONG inSmallSize, LONG inSmallEsize, LONG i
 #ifdef WITHZLIB
 	sort->file.ziosize = IObuffersize*sizeof(WORD);
 	sort->file.ziobuffer = 0;
+#endif
+	FILEHANDLE *stage4 = &(sort->FoStage4);
+	memset(stage4,0,sizeof(*stage4));
+	stage4->POsize = sort->file.POsize;
+	stage4->handle = -1;
+#ifdef WITHPTHREADS
+	stage4->pthreadslock = dummylock;
+#endif
+#ifdef WITHZLIB
+	stage4->ziosize = sort->file.ziosize;
+	stage4->ziobuffer = 0;
 #endif
 	if ( AM.S0 != 0 ) {
 		sort->file.name = (char *)(sort->file.PObuffer + IObuffersize);