Friday, 23 January 2015

Remove duplicate lines in a text file using php

Using this we can read and remove duplicate lines in a text file. Set the file permission of text file to 777 in linux.
<?php
function remove_lines($Filepath, $IgnoreCase=false, $NewLine="n"){

if (!file_exists($Filepath)){
$ErrorMsg = 'RemoveDuplicatedLines error: ';
$ErrorMsg .= 'The given file ' . $Filepath . ' does not exist!';
die($ErrorMsg);
}

$Content = file_get_contents($Filepath);

$Content = remove_lines_string($Content, $IgnoreCase, $NewLine);

if (!is_writeable($Filepath)){ //checks whether the file is writeable
$ErrorMsg = 'RemoveDuplicatedLines error: ';
$ErrorMsg .= 'The given file ' . $Filepath . ' is not writeable!';
die($ErrorMsg);
}

$FileResource = fopen($Filepath, 'w ');
fwrite($FileResource, $Content);
fclose($FileResource);
}

function remove_lines_string($Lines, $IgnoreCase=false, $NewLine="n"){

if (is_array($Lines))
$Lines = implode($NewLine, $Lines);

$Lines = explode($NewLine, $Lines);

$LineArray = array();

$Duplicates = 0;

for ($Line=0; $Line < count($Lines); $Line ){

$CurrentLine = trim($Lines[$Line]); //removes empty spaces

if ($CurrentLine == '')
continue;

$LineKey = $CurrentLine;

if ($IgnoreCase)
$LineKey = strtolower($LineKey);

if (!isset($LineArray[$LineKey]))
$LineArray[$LineKey] = $CurrentLine;
else
$Duplicates ;
}

asort($LineArray);

return implode($NewLine, array_values($LineArray));
}
//set the file permission of text.txt to 777 in Linux
$remlines = remove_lines("test.txt");
echo $remlines;
?>

0 comments:

Post a Comment