Tips_and_Hacks/convert.py

#version 0.2
#by John Wood - for Tech Advance
# This script reads through the monolithic .md files in a DokuWiki copy of OBS and splits them out into chunks in folders
# named like the .md files. The chunks are named 1.txt to n.txt where `n` is the last chunk. The folders and files are
# automatically named correctly. Because the script doesn't generate a manifest.json I create a project in translationStudio
# and then merge the folders created by this script into the translation folder created by translationStudio.

# Usage: python convert.py <path to DokuWiki OBS files>

# Import necessary python components
import os
# os is used for file system commands
import re
# re is used for regular expressions
import sys
# sys is used for command line arguments
import shutil
# shutil is high-level file operations

from subprocess import call
# to fork for git

# print "Starting the conversion process"

program_name=sys.argv[0];
arguments=sys.argv[1:];
count_args=len(arguments);
if count_args !=1 :
    # print "Usage: convert old_dir"
    sys.exit(1);
    
convertdir = sys.argv[1];
projectinfo = convertdir.split("_");
oldpath=projectinfo[0];
path=oldpath.split("/");
language=path[len(path)-1];
book=projectinfo[1];

#projectType=projectinfo[2];
if len(projectinfo)!=2:
    print"This may not be a DokuWiki OBS project, and I'm not sure how to handle it";
    sys.exit(1)
elif book=="obs":
    # munge the filename to get the name of the resulting folder.
    targetpath=convertdir+"_text_obs"
    
# print "The language is "+language;
# print "Source directory is "+convertdir;
# print "Target directory is "+targetpath;

# Create the manifest file

for filename in os.listdir(convertdir+"/content/"):
  #run the script in the target directory
    # Check for special folders first
    if (filename=="front"):
        if not os.path.exists(targetpath+"/front"):
            os.makedirs(targetpath+"/front")
        for filename2 in os.listdir(convertdir+"/content/"+filename+"/"):
            shutil.copyfile(convertdir+"/content/"+filename+"/"+filename2,targetpath+"/front/"+filename2.replace(".md",".txt"));
    # Else, if the filename is a MarkDown file
    elif filename.endswith(".md"):
        filenum = 00;
        # We start by making a folder/directory matching the name of the Markdown file
        newpath = filename.replace(".md","");
        newpath=targetpath+"/"+newpath;
        if not os.path.exists(newpath):
            os.makedirs(newpath)
            
        worksite = convertdir + "/content/";
        filename=worksite+filename;
        
        # Then, opening the file to scan it
        with open(filename) as mdfile:
            # Parsing the file by line
            for line in mdfile:
                # print "Working with line: "+line.strip("\s")
                # Lines with ![Image are image references not needed in the translationStudio project
                if re.match('\!\[Image|\[\[https',line):
                    # print " Found a line matching ![Image"
                    filenum = filenum + 1
                    # print "  Filenumber is "+format(filenum)
                elif re.match("#",line):
                    # print "   Found a line Matching # -- this is a Title"
                    # print "    Writing to "+newpath;
                    myTitle = newpath + "/title.txt"
                    with open(myTitle, "a+") as newfile:
                        # We write the same line to the new file, but erase the hashes (#)
                        newfile.write(line.replace("#",""));
                elif re.match("_",line):
                    # Lines with underscores are the references
                    myRef = newpath + "/reference.txt"
                    with open(myRef, "a+") as newfile:
                        #we write the same line to the new file, but erase the underscores
                        newfile.write(line.replace("_",""));
                # Only process lines that aren't blank
                elif not line.strip("\s")=="\n":
                    # print "Line "+'"'+line.strip("\s")+'"'+" is blank"
                #else:
                    # print "     The current file number is "+format(filenum)
                    if filenum==0: filenum=1
                    newFileName = "{:0>2d}".format(filenum)+".txt";
                    # print "      Writing the text file - "+newFileName
                    myNewFile = newpath + "/"+newFileName
                    with open(myNewFile, "a+") as newfile:
                        newfile.write(line)

print "Generating the manifest"
with open(convertdir+"/manifest.yaml") as manfile:
    with open(targetpath+"/manifest.json","a+") as newmanfile:
        for manline in manfile:
            title_match = re.search(r"^    title: (.+)",manline)
            direction_match = re.search(r"    direction: (\w+)",manline)
            modified_date_match= re.search(r"  modified: '(\d\d\d\d-\d\d-\d\d)'",manline)
            if modified_date_match:
                modified_date=modified_date_match.group(1);
            elif title_match:
                target_language_name=title_match.group(1);
            elif direction_match:
                target_direction = direction_match.group(1);
                
        newmanfile.write('\n'.join([
            '{',
            '	"package_version": 6,',
            '	"format": "markdown",',
            '	"generator": {',
            '		"name": "ts-desktop",',
            '		"build": "132"',
            '	},',
            '	"target_language": {',
            '		"id": "'+language+'",',
            '		"name": "'+target_language_name+'",',
            '		"direction": "'+target_direction+'"',
            '	},',
            '	"project": {',
            '		"id": "obs",',
            '		"name": "Open Bible Stories"',
            '	},',
            '	"type": {',
            '		"id": "text",',
            '		"name": "Text"',
            '	},',
            '	"resource": {',
            '		"id": "obs",',
            '		"name": "Open Bible Stories"',
            '	},',
            '	"source_translations": [',
            '		{',
            '			"language_id": "en",',
            '			"resource_id": "obs",',
            '			"checking_level": "3",',
            '			"date_modified": "'+modified_date+'",',
            '			"version": "4"',
            '		}',
            '	],',
            '	"parent_draft": {},',
            '	"translators": [],',
            '	"finished_chunks": []',
            '}']));
            
#gitCommand = "/usr/bin/git init "+targetpath;
#call("/usr/bin/git init", targetpath)