[Данные, строки] Удалить из файла все повторения

$sOrigin = @ScriptDir & "\file1.txt"
$sResulut = @ScriptDir & "\file2.txt"
$hRead = FileOpen($sOrigin, 0)
$hWrite = FileOpen($sResulut, 2)
$sBuffer = ''
$lineSum = ''
While 1
        $line = FileReadLine($hRead)
        If @error = -1 Then ExitLoop
        If Not StringInStr($lineSum, $line) Then
            $sBuffer &= $line & @CRLF
            $lineSum &= $line & "|"
        EndIf
WEnd
FileClose($hRead)
FileWrite($hWrite, $sBuffer)
FileClose($hRead)

FileClose($hWrite)

#include <file.au3>

Dim $aLines,$sSum

$sFile = @ScriptDir & "\Test.txt"
_FileReadToArray($sFile,$aLines)
$sText= FileRead($sFile)

For $i=UBound($aLines)-1 To 1 Step -1
   If StringRegExp($sSum,'(?m)\n\r?' & $aLines[$i] & '\n\r?',0) Then $iProc= _FileWriteToLine($sFile, $i,'',1); Корректен
   ; If StringRegExp($sSum,$aLines[$i]&'\n\r?',0) Then $iProc= _FileWriteToLine($sFile, $i,'',1); оказалось не совсем корректен
   ;If StringRegExp($sSum,$aLines[$i],0) Then $iProc= _FileWriteToLine($sFile, $i,'',1) ; некорректен
   ;If StringInStr($sSum,$aLines[$i]) Then $iProc= _FileWriteToLine($sFile, $i,'',1);некорректен
   $sSum &= $aLines[$i]& @LF
Next

If StringRegExp($sSum,$aLines[$i]&'\n\r?',0) Then $iProc= _FileWriteToLine($sFile, $i,'',1); Оказалось не рассматривает начало и конец текста

If StringRegExp($sSum,'(?m)\n\r?' & $aLines[$i] & '\n\r?',0) Then $iProc= _FileWriteToLine($sFile, $i,'',1)

#include <file.au3>
#include <Array.au3>

Dim $aFileNew[1], $afileOld
$u = 0
$timeout = TimerInit()
_FileReadToArray("111.txt", $afileOld)
For $i = 1 To UBound($afileOld) - 1
	_ArraySearch($aFileNew, $afileOld[$i])
	If @error = 6 Then
		_ArrayAdd($aFileNew, $afileOld[$i])
		$u += 1
		$aFileNew[0] = $u
	EndIf
Next
_FileWriteFromArray("222.txt", $aFileNew, 1)
$timeout = Round(TimerDiff($timeout))
MsgBox(0, "", $timeout / 1000 & "sec")

#include <file.au3>

Dim $aLines,$sSum

$sFile = @ScriptDir & "\Test.txt"
_FileReadToArray($sFile,$aLines)

For $i=UBound($aLines)-1 To 1 Step -1
   If StringRegExp($sSum,'(?m)\n\r?' & $aLines[$i] & '\n\r?',0) Then 
     $iProc= _FileWriteToLine($sFile, $i,'',1)
     ContinueLoop
   EndIf
   $sSum &= $aLines[$i]& @LF
Next

$sFile = @DesktopDir & "\Test.txt"

#Region Create file (30+- kb)
$sData = ""
$iRandom1 = Random(1, 50, 1)
$iRandom2 = Random(1, 50, 1)

For $i = 1 To 1024*5 ;1024*1024 ;7+ mb
	$sData &= $i & @CRLF
	
	If $i = $iRandom1 Or $i = $iRandom2 Then
		$sData &= $i & @CRLF
		ConsoleWrite($i & @LF) ;Display the duplicates
	EndIf
Next

$hFile = FileOpen($sFile, 2)
FileWrite($hFile, $sData)
FileClose($hFile)
#EndRegion Create file (30+- kb)

$Ret = _FileDeleteDuplicates($sFile, 1)

If $Ret = 1 Then
	MsgBox(64, "Results", "Was deleted <" & @extended & "> duplicates in file [" & $sFile & "]")
ElseIf @error Then
	MsgBox(48, "Error", "File [" & $sFile & "] was not found.")
Else
	MsgBox(64, "Results", "No duplicates found.")
EndIf

Func _FileDeleteDuplicates($sFile, $sDirection=-1)
	If Not FileExists($sFile) Or StringInStr(FileGetAttrib($sFile), "D") Then Return SetError(1, 0, 0)
	
	Local $FileStr = FileRead($sFile), $FileHasEndCRLF = True, $Extended = 0
	Local $ReplDirectVal = '\1\2'
	If $sDirection <> -1 Then $ReplDirectVal = '\2\1'
	
	If StringRight($FileStr, 2) <> @CRLF Then
		$FileStr &= @CRLF
		$FileHasEndCRLF = False
	EndIf
	
	Do
		$FileStr = StringRegExpReplace($FileStr, '(?sm)(^[^\r\n]+\r\n?)(.*?)^\1', $ReplDirectVal)
		$Extended += @extended
	Until @extended = 0
	
	If $Extended = 0 Then Return SetExtended(0, 0)
	
	If Not $FileHasEndCRLF Then $FileStr = StringTrimRight($FileStr, 2)
	
	Local $hFile = FileOpen($sFile, 2)
	FileWrite($hFile, $FileStr)
	FileClose($hFile)
	
	Return SetExtended($Extended/2, 1)
EndFunc

$sOrigin = @ScriptDir & "\111.txt"
$sResulut = @ScriptDir & "\222.txt"
$timeout = TimerInit()
$hRead = FileOpen($sOrigin, 0)
$hWrite = FileOpen($sResulut, 2)
$sBuffer = ''
$lineSum = ''
While 1
        $line = FileReadLine($hRead)
        If @error = -1 Then ExitLoop
        If StringRegExp ($lineSum, '(?m)\|' & $line & '\|', 0 )=0 Then
       ;If Not StringInStr($lineSum, $line ) Then
            $sBuffer &= $line & @CRLF
            $lineSum &= $line & "|"
        EndIf
WEnd
FileClose($hRead)
FileWrite($hWrite, $sBuffer)
FileClose($hRead)

$timeout = Round(TimerDiff($timeout))
MsgBox(0, "", $timeout / 1000 & "sec")

$sOrigin = @ScriptDir & "\file1.txt"
$sResulut = @ScriptDir & "\file2.txt"
$hRead = FileOpen($sOrigin, 0)
$hWrite = FileOpen($sResulut, 2)
$sBuffer = ''
$lineSum = '|'
While 1
	$line = FileReadLine($hRead)
	If @error = -1 Then ExitLoop
	If Not StringInStr($lineSum, "|" & $line & "|") Then $sBuffer &= $line & @CRLF
	$lineSum &= $line & "|"
WEnd
FileClose($hRead)
FileWrite($hWrite, $sBuffer)
FileClose($hWrite)

$aLines = StringSplit(StringStripCR(FileRead($sFile)), @LF)
$sData = @CRLF

For $i = 1 To $aLines[0]
	If Not StringInStr($sData, @CRLF & $aLines[$i] & @CRLF, 1, -1) Then
		$sData &= $aLines[$i] & @CRLF
	Else
		ConsoleWrite($i & @LF)
	EndIf
Next

$sData = StringTrimLeft($sData, 2)
$sData = StringTrimRight($sData, 2)

$hFile = FileOpen($sFile, 2)
FileWrite($hFile, $sData)
FileClose($hFile)