# -*- coding: utf-8 -*-
"""
Created on Mon Mar  4 10:48:00 2019

@author: MB
"""
# =============================================================================
# Libraries
# =============================================================================
import os
import csv
from os import listdir
from os.path import isfile, join
import regex


# =============================================================================
# Useful functions
# =============================================================================
#function for removing empty rows from a list
def remove_empty_rows_list(A):
    list(filter(None,A))
#function for removing empty rows from a file
def remove_empty_lines(filename):
    #Overwrite the file, removing empty lines and lines that contain only whitespace.
    with open(filename,encoding='utf-8-sig') as in_file, open(filename,'r+',encoding='utf-8-sig') as out_file:
        out_file.writelines(line for line in in_file if line.strip())
        out_file.truncate() 

# =============================================================================
# Openning the files
# =============================================================================

#Openning the files
Article_files= [ f for f in listdir(r'''G:\Developpement\Déroulement-- Implémentation\Corpus\Wikipedia\arwiki files - to CSV\Csvs''') if isfile(join(r'''G:\Developpement\Déroulement-- Implémentation\Corpus\Wikipedia\arwiki files - to CSV\Csvs''',f))]
path="G:/Developpement/Déroulement-- Implémentation/Corpus/Wikipedia/arwiki files - to CSV/Csvs"

#creating a list of articles
list_of_articles=[]

for file in Article_files:
    os.chdir(path)
    with open(file,'r',encoding='utf-8-sig') as f:
        print('22222222222222222222222222222222222222222')
        reader=csv.reader(row.replace('/0','') for row in f)
        print('333333333333333333333333333333333333333333')
        Articles=[]
        print('44444444444444444444444444444444444444444444')
        Articles=list(reader)
        print('555555555555555555555555555555555555555555555')
        list_of_articles.append(Articles)
        print('Success')

remove_empty_rows_list(list_of_articles)    

# =============================================================================
# Cleaning the files in a list
# =============================================================================
#save all the articles in one file
os.chdir("G:/Developpement/Déroulement-- Implémentation/Corpus/Wikipedia/arwiki files - to CSV")
with open('wiki_articles.csv','a',encoding='utf-8-sig') as savefile:
    for row in list_of_articles:
        for cell in row:
            wr=csv.writer(savefile,quoting=csv.QUOTE_ALL)
            wr.writerow(cell)
            savefile.write('\n')
remove_empty_lines('wiki_articles.csv')            

list_articles=[]
for row in list_of_articles:
    for cell in row:
        list_articles.append(cell)

remove_empty_rows_list(list_articles)
#Remove non Arabic words from the list/file using the regex function
#with open('wiki_articles.csv','r',encoding='utf-8-sig') as articles:
 #   reader=csv.reader(row.replace('\0','')for row in articles)
 #   list_articles=list(reader)

list_of_arabic_articles=[]
for row in list_articles:
    for cell in row:
            list_of_arabic_articles.append(regex.sub('[^\p{Arabic}]',' ',cell))
            print('success')

remove_empty_rows_list(list_of_arabic_articles)

#Normalization
A='[آ]|[أ]|[إ]'
B='[ؤ]|[ئ]'
C='[ة]'
D='[ي]|[ى]'
r='أ'
u='ء'
v='ه'
w='ي'
def normalizing(string):
        string=regex.sub('[آ]|[أ]|[إ]',r,string)
        string=regex.sub('[ؤ]|[ئ]',u,string)
        string=regex.sub('[ة]',v,string)
        string=regex.sub('[ي]|[ى]',w,string)
        return string

Normalized_Articles=[]
for row in list_of_arabic_articles:
        Normalized_Articles.append(normalizing(row))

# =============================================================================
# Saving the files in one csv file
# =============================================================================

with open('Arabic_clean_wiki.csv','a',encoding='utf-8-sig') as savefile:
    for row in Normalized_Articles:
        savefile.write(row)
# =============================================================================
# this part is for the streemes articles not the downloaded ones
# =============================================================================
#list the file
wiki_summaries=[]
os.chdir("G:\Developpement\Déroulement-- Implémentation\Corpus\Wikipedia")
with open('wikipidea_summaries.csv','r',encoding='utf-8-sig') as infile:
    reader=csv.reader(row.replace('/0','')for row in infile)
    wiki_summaries=list(reader)
#cleaning the list
#removing the non_arabic characters
list_of_arabic_summaries=[]
for row in wiki_summaries:
    for cell in row:
        list_of_arabic_summaries.append(regex.sub('[^\p{Arabic}]',' ',cell))

#normalizing the texts
normalized_summaries=[]
for row in list_of_arabic_summaries:
    normalized_summaries.append(normalizing(row))

with open('Arabic_clean_wiki.csv','a',encoding='utf-8-sig') as savefile:
    for row in normalized_summaries:
        savefile.write(row)

remove_empty_lines('Arabic_clean_wiki.csv')