;this Function will load the contents of a text file into an array
Function loadfile($file)
	DIM $fso,$f,$fs
	$fso = CreateObject("Scripting.FileSystemObject")
	$f = $fso.GetFile($file)
	If @ERROR Exit(2) EndIf
	$fs = $f.OpenAsTextStream(1)
	$loadfile = Split($fs.Read($f.size),@CRLF)
	Exit(@ERROR)
EndFunction

Quote:

---

Actualy his readfile() UDF is using the OPEN() and Readline() commands... so i doubt that the CPU usage will drop when dealing with such a LARGE file.

---

No, that's not correct. Look at the udf more closely, and you will see the obvious problems.

To elaborate, there are two major inefficiencies in the ReadFile() udf.

The most obvious is where you specify a postive limit on the number of lines to be returned. In this case the entire file is read before the code checks to see if there is a limit. This means that if you need the first 10 lines from your 10,000 line file all 10,000 lines will be read anyway.

The second less obvious problem is how ReadFile() manipulates the data.

As each line is read from the file buffer, it is catenated to the end of a string. This string gets longer and longer until it eventually becomes the size of your file. As you can imagine, string manipulations on data this size become slower as the string gets larger, and with a 800KB text file there is likely to be a large number of iterations of the ReadLine() loop.

The string is then converted to an array - again this is not a trivial task for a string of this length.

All this string manipulation occurs in core memory and requires repeated creation and destruction of temporary variable / stack space in the scripts private process space.

The string maniplation is where the majority CPU cycles are burned, not on the file IO.

The reason for moving the ReadLine() activity to the main script is that the temporary string is not created so we lose the massive overhead. As we need to iterate the result anyway we also save the overhead of the do..until loop in the UDF.

However, FSO is a good option too. In this case even with the Split() it may well be quicker than using KiXtart file IO

[edit]
Contribution to the Secret Long Line Policemans ball to follow...
[/edit]

Quote:

---

Nice code Bryce, except for the use of parens to force precedence on the Exit parm where none is needed.

---

I wrote that back before the EXIT police started their crack down.....

Function ReadFile($file)

	Dim $i,$f,$L

	$f = FreeFileHandle

	$i = Open($f,$file)

	If @Error

		Exit @Error

	EndIf

	$i = 0

	$L = ReadLine($f)

	While @Error = 0

		ReDim Preserve $ReadFile[$i]

		$ReadFile[$i] = $L

		$i = $i + 1

		$L = ReadLine($f)

	Loop

	$i = Close($f)

	Exit VarType($ReadFile) = 0 ; Exitcode 1 if file is empty

EndFunction

$start = cdbl(""+flipctime(@date,@time)+"."+@msecs)
$file = loadfile("dir.txt")
$end = cdbl(""+flipctime(@date,@time)+"."+@msecs)
? "Loadfile() = " $end - $start

$start = cdbl(""+flipctime(@date,@time)+"."+@msecs)
$file = readfile("dir.txt")
$end = cdbl(""+flipctime(@date,@time)+"."+@msecs)
? "ReadFile() = " $end - $start



Function ReadFile($file)
	Dim $i,$f,$L
	$f = FreeFileHandle
	$i = Open($f,$file)
	If @Error
		Exit @Error
	EndIf
	$i = 0
	$L = ReadLine($f)
	While @Error = 0
		ReDim Preserve $ReadFile[$i]
		$ReadFile[$i] = $L
		$i = $i + 1
		$L = ReadLine($f)
	Loop
	$i = Close($f)
	Exit VarType($ReadFile) = 0 ; Exitcode 1 if file is empty
EndFunction

;this Function will load the contents of a text file into an array
Function loadfile($file)
	DIM $fso,$f,$fs
	$fso = CreateObject("Scripting.FileSystemObject")
	$f = $fso.GetFile($file)
	If @ERROR Exit(2) EndIf
	$fs = $f.OpenAsTextStream(1)
	$loadfile = Split($fs.Read($f.size),@CRLF)
	Exit(@ERROR)
EndFunction

Function FlipcTime($date,$time,optional $tz)
	dim $y,$m,$d
	$date = split($date,"/")
	if ubound($date) <> 2 exit(1) endif
	$y=val($date[0]) $m=val($date[1]) $d=val($date[2])
	if $m<3
		$m=$m+12
		$y=$y-1
	endif
	$Date=$d+(153*$m-457)/5+365*$y+$y/4-$y/100+$y/400-306
	$time = split($time,":")
	select 
		case ubound($time)=1
			redim preserve $time[2]
			$time[2]=0
		case ubound($time)=2
		case 1
			exit(1)
	endselect
	$time = (val($time[0])*3600)+(val($time[1])*60)+val($time[2])
	$flipctime = IIF($tz,(($date-719163)*86400 + $time)-($tz*3600),($date-719163)*86400 + $time)
endfunction

C:\temp>kix32 readfile.kix

Loadfile() = 0.75
ReadFile() = 1.42199993133545
C:\temp>kix32 readfile.kix

Loadfile() = 0.75
ReadFile() = 1.42200016975403
C:\temp>kix32 readfile.kix

Loadfile() = 0.75
ReadFile() = 2.12400007247925
C:\temp>kix32 readfile.kix

Loadfile() = 0.765000104904175
ReadFile() = 1.42199993133545
C:\temp>kix32 readfile.kix

Loadfile() = 0.766000032424927
ReadFile() = 1.42199993133545
C:\temp>

break on

dim $start, $arrInc, $inc, $file

$arrInc = 1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192

for each $inc in $arrInc
	dim $file

	$start=@TICKS
	$file = ""
	$file = loadfile("dir.txt")
	$file = ""
	? "Loadfile() = " @TICKS - $start

	chr(9)

	$start=@TICKS
	$file = ""
	$file = readfile("dir.txt", $inc, 0)
	$file = ""

	"constant increment ReadFile("+$inc+") = " @TICKS - $start

	chr(9)

	$start=@TICKS
	$file = ""
	$file = readfile("dir.txt", $inc, 1)
	$file = ""

	"variable increment ReadFile("+$inc+") = " @TICKS - $start
next
?
exit

; will load the contents of a text file into an array only with Kix
Function ReadFile($file, optional $inc, optional $incvariable )
	Dim $, $i,$f,$L, $j

	if not $inc	$inc = 1	endif

	Dim $ReadFile[$inc]

	$f = FreeFileHandle()
	$ = Open($f,$file)
	if @Error
		Exit @Error
	endIf

	$i = -1
	$j = 0
	$L = ReadLine($f)
	While not @Error
		$i = $i + 1
		if $i > $j
			$j = $j + $inc
			ReDim Preserve $ReadFile[$j]
			if $incvariable
				$inc = $inc + $inc
			endif
		endif
		$ReadFile[$i] = $L
		$L = ReadLine($f)
	Loop
	$ = Close($f)
	if $i=-1
		$ReadFile = ""
		exit 1
	else
		redim preserve $ReadFile[$i]
		exit 0
	endif
EndFunction

; will load the contents of a text file into an array with WScript
Function loadfile($file)
	DIM $, $fso,$f,$fs
	$fso = CreateObject("Scripting.FileSystemObject")
	$f = $fso.GetFile($file)
	If @ERROR Exit(2) EndIf
	$fs = $f.OpenAsTextStream(1)
	$loadfile = Split($fs.Read($f.size),@CRLF)
	$ = $f.Close
	Exit(@ERROR)
EndFunction

Loadfile() = 961       constant increment ReadFile(1) = 1482        variable increment ReadFile(1) = 651
Loadfile() = 1042      constant increment ReadFile(2) = 1372        variable increment ReadFile(2) = 671
Loadfile() = 1091      constant increment ReadFile(4) = 1282        variable increment ReadFile(4) = 671
Loadfile() = 1142      constant increment ReadFile(8) = 1201        variable increment ReadFile(8) = 691
Loadfile() = 1232      constant increment ReadFile(16) = 1122       variable increment ReadFile(16) = 741
Loadfile() = 1272      constant increment ReadFile(32) = 1191       variable increment ReadFile(32) = 731
Loadfile() = 1262      constant increment ReadFile(64) = 1042       variable increment ReadFile(64) = 741
Loadfile() = 1262      constant increment ReadFile(128) = 961       variable increment ReadFile(128) = 761
Loadfile() = 1272      constant increment ReadFile(256) = 821       variable increment ReadFile(256) = 731
Loadfile() = 1352      constant increment ReadFile(512) = 791       variable increment ReadFile(512) = 741
Loadfile() = 1292      constant increment ReadFile(1024) = 771      variable increment ReadFile(1024) = 731
Loadfile() = 1282      constant increment ReadFile(2048) = 751      variable increment ReadFile(2048) = 741
Loadfile() = 1242      constant increment ReadFile(4096) = 801      variable increment ReadFile(4096) = 761
Loadfile() = 1342      constant increment ReadFile(8192) = 842      variable increment ReadFile(8192) = 761

Dim $ReadFile[$inc] 

to

reDim $ReadFile[$inc]

getfilesize(c:\dir.txt)=7725170
Loadfile() = 2391       CI ReadFile(1) = 6890   VI ReadFile(1) = 2781
Loadfile() = 2766       CI ReadFile(2) = 6578   VI ReadFile(2) = 2922
Loadfile() = 2844       CI ReadFile(4) = 6265   VI ReadFile(4) = 2969
Loadfile() = 2875       CI ReadFile(8) = 6125   VI ReadFile(8) = 2953
Loadfile() = 2922       CI ReadFile(16) = 6078  VI ReadFile(16) = 3078
Loadfile() = 2907       CI ReadFile(32) = 6015  VI ReadFile(32) = 3016
Loadfile() = 2891       CI ReadFile(64) = 5984  VI ReadFile(64) = 3031
Loadfile() = 2906       CI ReadFile(128) = 6157 VI ReadFile(128) = 3109
Loadfile() = 2891       CI ReadFile(256) = 6093 VI ReadFile(256) = 3141
Loadfile() = 2938       CI ReadFile(512) = 4640 VI ReadFile(512) = 3094
Loadfile() = 2906       CI ReadFile(1024) = 3781        VI ReadFile(1024) = 3094
Loadfile() = 2891       CI ReadFile(2048) = 3453        VI ReadFile(2048) = 3141
Loadfile() = 2906       CI ReadFile(4096) = 3297        VI ReadFile(4096) = 3125
Loadfile() = 2968       CI ReadFile(8192) = 3172        VI ReadFile(8192) = 3110