_FileDeleteDuplicateLines - Удаление дубликатов в файле

#include <Array.au3>

$sFileName = 'Test.txt'
$sFilePath = @DesktopDir
$sFile = $sFilePath & '\' & $sFileName

$aDups = _FileDeleteDuplicateLines($sFile, 0, 1)

Switch @error
	Case 1
		MsgBox(48, 'Ошибка', StringFormat('Файл <%s> не найден.', $sFileName))
	Case 2
		MsgBox(48, 'Ошибка', StringFormat('Ошибка при попытке записи в Файл <%s>.', $sFileName))
	Case Else
		If IsArray($aDups) Then
			MsgBox(64, 'Results', StringFormat('В файле <%s> удалено дублирующихся строк: %i', $sFileName, $aDups[0]))
			_ArrayDisplay($aDups, 'Duplicate Lines')
		Else
			MsgBox(64, 'Results', StringFormat('В файле <%s> нет дублирующихся строк.', $sFileName))
		EndIf
EndSwitch

; #FUNCTION# ====================================================================================================
; Name...........:	_FileDeleteDuplicateLines
; Description....:	Deletes duplicated lines in file
; Syntax.........:	_FileDeleteDuplicateLines($sFile [, $iCaseSense = 0 [, $iUseDict = 0 ]])
; Parameters.....:	$sFile - File to delete the duplicates in.
;					$iCaseSense - [Optional] Defines if the search for duplicates will be case sensitive (default is 0 - not case sensitive).
;					$iUseDict - [Optional] If this parameter is 1 (default is 0), then Scripting.Dictionary object is used to find and delete the duplicates (much faster).
;											Note: This method works only on Win 2000 and above.
;					
; Return values..:	Success - Return an array of deleted lines ([0] is the number of total deleted duplicates).
;					Failure - Set @error as following:
;														-1 - If $iUseDict = 1 and @OSTYPE = 'WIN32_WINDOWS'.
;														1 - $sFile does not exists.
;														2 - Unable to write the file.
; Author.........:	G.Sandler (CreatoR)
; Modified.......:	
; Remarks........:	
; Related........:	
; Link...........:	
; Example........:	
; ===============================================================================================================
Func _FileDeleteDuplicateLines($sFile, $iCaseSense = 0, $iUseDict = 0)
	If Not FileExists($sFile) Then
		Return SetError(1, 0, 0)
	EndIf
	
	If $iUseDict And @OSTYPE = 'WIN32_WINDOWS' Then
		Return SetError(-1, 0, 0)
	EndIf
	
	Local $aDups, $sFRead, $aFSplit, $sFContent, $oDict, $hFOpen
	
	$sFRead = FileRead($sFile)
	$aFSplit = StringSplit(StringStripCR($sFRead), @LF)
	
	Dim $aDups[$aFSplit[0]+1]
	
	If $iUseDict Then
		$oDict = ObjCreate('Scripting.Dictionary')
		
		If $iCaseSense = 1 Then
			$oDict.CompareMode = 0 ;Binary mode (case sensitive)
		Else
			$oDict.CompareMode = 1 ;Text mode (not case sensitive)
		EndIf
		
		For $i = 1 To $aFSplit[0]
			If Not $oDict.Exists($aFSplit[$i]) Then
				$oDict.Add($aFSplit[$i], $aFSplit[$i])
				$sFContent &= $aFSplit[$i] & @CRLF
			Else
				$aDups[0] += 1
				$aDups[$aDups[0]] = $aFSplit[$i]
			EndIf
		Next
	Else
		$sFContent = @CRLF
		
		If $iCaseSense = 0 Then
			$iCaseSense = 2
		EndIf
		
		For $i = 1 To $aFSplit[0]
			If Not StringInStr($sFContent, @CRLF & $aFSplit[$i] & @CRLF, $iCaseSense) Then
				$sFContent &= $aFSplit[$i] & @CRLF
			Else
				$aDups[0] += 1
				$aDups[$aDups[0]] = $aFSplit[$i]
			EndIf
		Next
		
		$sFContent = StringTrimLeft($sFContent, 2)
	EndIf
	
	If $aDups[0] = 0 Then
		Return 0
	EndIf
	
	ReDim $aDups[$aDups[0]+1]
	
	$sFContent = StringTrimRight($sFContent, 2)
	
	$hFOpen = FileOpen($sFile, 2)
	
	If $hFOpen = -1 Then
		Return SetError(2, 0, 0)
	EndIf
	
	FileWrite($hFOpen, $sFContent)
	FileClose($hFOpen)
	
	Return $aDups
EndFunc

#include <Array.au3>

$sText = FileRead(@ScriptDir & '\тест.txt')
$timer1 = TimerInit()
$sText1 = _ArrayRemoveDuplicates($sText)
$timer1 = Round(TimerDiff($timer1), 2)

$timer2 = TimerInit()
$sText2 = _DeleteDuplicateLines($sText)
$timer2 = Round(TimerDiff($timer2), 2)

MsgBox(0, "Время выполнения", _
		'$timer1 : ' & $timer1 & @CRLF & _
		'$timer2 : ' & $timer2)
; MsgBox(0, 'Сообщение', $sText1)
; MsgBox(0, 'Сообщение', $sText2)

Func _ArrayRemoveDuplicates(Const ByRef $sTXT)
	Local $oDict = ObjCreate("Scripting.Dictionary")
	$oDict.CompareMode = 0
	$aFSplit = StringSplit(StringStripCR($sTXT), @LF)
	For $i In $aFSplit
		$oDict.Item($i)
	Next
	$sText = ''
	$aFSplit = $oDict.Keys()
	For $i In $aFSplit
		$sText &= $i & @CRLF
	Next
	Return $sText
EndFunc   ;==>_ArrayRemoveDuplicates

Func _DeleteDuplicateLines(Const ByRef $sTXT)
	Local $oDict = ObjCreate("Scripting.Dictionary")
	$oDict.CompareMode = 0
	$aFSplit = StringSplit(StringStripCR($sTXT), @LF)
	$sText = ''
	For $i = 1 To $aFSplit[0]
		If Not $oDict.Exists($aFSplit[$i]) Then
			$oDict.Add($aFSplit[$i], $aFSplit[$i])
			$sText &= $aFSplit[$i] & @CRLF
		EndIf
	Next
	Return $sText
EndFunc   ;==>_DeleteDuplicateLines