use python for removing bad symbols

This commit is contained in:
flukx 2021-01-11 01:59:31 +00:00
parent 7233857134
commit 339b84522f
4 changed files with 246 additions and 421 deletions

View file

@ -5,7 +5,7 @@
* [`.sh`-script for renaming pictures. Use date for filename.](addDateTaken.sh)
* [Python script for getting descriptions of OSM-notes from `.osn`-files. Also see "Tricks and tutorials"](extract-hinweise.py)
* [`.sh`-script for finding and listing all symlinks inkluding the linked paths](listLinks.sh)
* [`.sh`-script for automatic renaming of files for removal of annoying symbols](removeBadSymbols.sh)
* [python script for automatic renaming of files for removal of annoying symbols](removeBadSymbols.py)
* [Python3 script for removal of all LaTeX-created files (clean up)](rmCreated.py)
* [convert `.wav` files into `.mp3` files via vlc](wavToMp3.sh)
* [Packing list (for all kinds of vacations) (german)](packliste.txt)

View file

@ -13,9 +13,7 @@
# -l --onlylog just logs what would have been done but not doing anything
# -o [file] --outputfile [file] specifies the output (log) file (standard is da.log)
# -v [file] --voutputfile [file] specifies the verbose output log file (standard dav.log)
# to be implemented in other sript:
# -i --image adds date taken in beginning of file name if jpg, .png, (not yet: .mp4, .mov, ...)
# -q --quiet do not log
#Functions
vLog()
@ -78,6 +76,7 @@ usage()
echo " -l, --onlylog just logs what would have been done but not doing anything"
echo " -o [file], --outputfile [file] specifies the output (log) file (standard is da.log)"
echo " -v [file], --voutputfile [file] specifies the verbose output log file (standard dav.log)"
echo " - q, --quiet do not log"
echo " -- finishes the list of options, next argument is taken as the dir"
#echo " -i, --image adds date taken in beginning of file name if jpg, .png, (not yet: .mp4, .mov, ...)"
echo ""
@ -134,6 +133,9 @@ while test -n "$1" ; do
shift
VLOGFILE="$(folderloc "$1")"
;;
-q | --quiet ) VLOGFILE="/dev/null"
NVLOGFILE="/dev/null"
;;
-a | --noask | --no-interactive ) ops="$ops -a"
ask=0
;;

240
removeBadSymbols.py Executable file
View file

@ -0,0 +1,240 @@
#! /usr/bin/env python3
# -*- coding: utf-8 -*-
"""Rename files with bad characters.
The shell often is complicated if files have weird name.
This script renames files in a directory tree so that they
do not include those bad characters. The worst are whitespace.
"""
from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
import logging
import os
import os.path
LOGGING_LEVELS = {"debug": logging.DEBUG,
"info": logging.INFO,
"warning": logging.WARNING,
"error": logging.ERROR
}
INVALID_NAMES = ["", ".", ".."]
# most common replacement string
REPLACER = "_"
SINGLE_SYMBOLS = r'ãāǎàčēéěèȩêėīíǐìĩïıōóǒòũūúǔùǖǘǚǜşļĻķĶḩģĢḨņŅŗŖĀǍÀĒÉĚÈĪÍǏÌŌÓǑÒŪÚǓÙǕǗǙǛý'
SINGLE_REPLACE_SYMBOLS = r'aaaaceeeeeeeiiiiiiioooouuuuuüüüüslLkKhgGHnNrRAAAEEEEIIIIOOOOUUUUÜÜÜÜy'
REPLACEMENTS = {r"'": "",
r'"': "",
r"`": "",
r"´": "",
r"&": REPLACER + "und" + REPLACER,
r"*": "x",
r"(with lyrics)": "",
r" ": REPLACER,
r"C#": "C_sharp",
r"c#": "c_sharp",
r"#": REPLACER,
r"|": "l",
r",": REPLACER,
r"{": "",
r"}": "",
r"(": "",
r")": "",
r"[": "",
r"]": "",
r"~": "-",
r":": REPLACER,
r"@": "-at-",
r"?": "",
r">": REPLACER,
r"<": REPLACER,
r"ä": "ae",
r"Ä": "Ae",
r"á": "au", # icelandic
r"Á": "Au",
r"ö": "oe",
r"Ö": "Oe",
r"ü": "ue",
r"Ü": "Ue",
r"": "SS",
r"ß": "ss",
r"ð": "dh",
r"Ð": "Dh",
r"þ": "th",
r"Þ": "Th",
r"μFSR": "myFSR",
r"μ": "mu",
r"$": "USD",
r"": "EUR",
r".jpeg": ".jpg",
r".JPG": ".jpg",
r".JPEG": ".jpg",
"\u202f": "_", # some space
"\\": REPLACER
}
REPLACEMENTS.update({s : SINGLE_REPLACE_SYMBOLS[i] for i, s in enumerate(SINGLE_SYMBOLS)})
for begin, end, replacement in [
('2000', '200F', REPLACER), # various spaces and formatters
('2010', '2017', '-'), # various dashes
('2018', '201F', ""), # various apostrophies
('2028', '202E', ""), # separators, format characters
('2032', '203A', "") # various apostrophies and quotation marks
# one could add many more
]:
REPLACEMENTS.update({'u' + hex(hexspace)[2:]: replacement # replacing 0x by u
for hexspace in range(int(begin, 16), int(end, 16) + 1)})
def parse_args():
"""Parse command line arguments.
Returns:
Dictionary with the command line arguments.
"""
parser = ArgumentParser(
description="""Remove bad characters by renaming files.
Which files are renamed is logged in a log file.
(See below in --log.)""",
formatter_class=ArgumentDefaultsHelpFormatter
)
parser.add_argument(
"-a", "--noask", "--no-interactive",
dest="ask",
help=("Specify if renaming should not be confirmed by the user" +
" but just done."),
action="store_false"
)
parser.add_argument(
"-l", "--onlylog",
dest="rename",
action="store_false",
help="Dry run."
)
parser.add_argument(
"--log", "--logfile",
dest="log",
metavar="L",
help="""the file where to write the logging output.
This file is overwritten if existing.
If not specified assume 'rename.log'""",
action="store",
default=os.path.join("..", "rename.log")
)
parser.add_argument(
"--loglevel", "--level",
dest="loglevel",
help="""Specify the log level.
'error' includes OS errors at renaming files.
'warning' additionally includes no-renames because of weird resulting file name.
'info' includes no-renames because of existing files and renames and working directories.
'debug' includes good files.""",
default="info",
choices=LOGGING_LEVELS.keys()
)
parser.add_argument(
nargs="?",
dest="top",
metavar="directory",
help="""The directory in which to rename files.""",
action="store",
default=".")
return parser.parse_args()
def transform_filename(name: str):
"""Remove all bad symbols from name."""
for symbol, replacement in REPLACEMENTS.items():
name = name.replace(symbol, replacement)
name.lstrip("-")
while True:
old_name = name
for separator in ["-", "_", "."]:
name = name.replace(REPLACER + separator, separator)
name = name.replace(separator + REPLACER, separator)
name = name.replace(separator + separator, separator)
if old_name == name:
break
return name
def rename_file(directory, file, args):
"""Rename file directory/file if renaming is OK."""
new_name = transform_filename(file)
path = os.path.join(directory, file)
new_path = os.path.join(directory, new_name)
if new_name == file:
logging.debug("'{}' is OK.".format(file))
return
if os.path.lexists(new_path):
logging.info("'{}' is not renamed to '{}' because this already exists.".format(
file, new_name
))
return
if new_name in INVALID_NAMES:
logging.warning("'{}' is not renamed because it would invalid: '{}'.".format(
path, new_name))
return
if args.ask:
rename = input("Rename '{}' to '{}'? (Enter for yes)".format(
path, new_name
))
rename = not rename.startswith("n")
else:
rename = True
if rename:
if args.rename: # == not onlylog
try:
os.rename(path, new_path)
except FileNotFoundError as error:
logging.error("Could not move '{}' to '{}' due to FileNotFoundError: {}".format(
path, new_name, error
))
logging.info("Rename '{}' to '{}'".format(file, new_name))
else:
logging.info("Did not rename '{}' to '{}' due to user choice.".format(
file, new_name
))
def if_ignore(filename):
"""Return if this filename should be ignored.
If it's a directory, this means that the directory content is ignored as well."""
return (filename.startswith('.') or (
filename.startswith('__') and ( # special python file
filename.endswith('__.py') or
filename.endswith('__.html') or
filename.endswith('__.txt')
)) or filename.endswith('.class')) # java class file
if __name__ == "__main__":
args = parse_args()
logging.basicConfig(
filename=args.log,
level=LOGGING_LEVELS[args.loglevel]
)
for dirpath, dirnames, filenames in os.walk(args.top, topdown=True):
print('.', end='')
logging.info("Go to directory '{}'".format(os.path.abspath(dirpath)))
for dir in dirnames:
if if_ifignore(dir):
logging.debug("Ignore dir '{}'".format(dir))
dirnames.remove(dir)
else:
rename_file(dirpath, dir, args)
for file in filenames:
if if_ignore(file):
logging.debug("Ignore file '{}'".format(file))
else:
rename_file(dirpath, file, args)
# remove empty log file:
logging.shutdown()
if os.stat(args.log).st_size == 0:
os.remove(args.log)

View file

@ -1,417 +0,0 @@
#!/bin/bash
# this is a shebang. It guarantees that this script is executed with bash, not the currently open shell
# problem with shanging to sh: I can't get test to work with * wildcards :(
#todo: also handle hidden files -- actually doable but not a good idea
#todo: bei jeder Aenderung auch usage (doc) anpassen
#todo: translate entire script into english
# options
# -h --help display help
# -a --noask --no-interactive do not prompt the user to confirm renaming
# -l --onlylog just logs what would have been done but not doing anything
# the following two MUST be present # TODO: not making them mandatory
# -o [file] --outputfile [file] specifies the output (log) file (standard is rn.log)
# -v [file] --voutputfile [file] specifies the verbose output log file (standard rnv.log)
# -rm removes files of the form AlbumArt_*, Folder.jpg, Thumbs.db, desktop.ini
# -c converts file names to lower case
#Constants
# weil im folgenden ein Apostroph vorkommt, kann man diesen String nicht einfach einfuegen
STRICHSTR=s/\'//g
# auch hier muss " escaped werden
ANFUERSTR=s/\"//g
# und hier ist es `
ANDSTRSTR=s/\`//g
# und hier ist es &
AMBERSSTR=s/\&/_und_/g
# und hier ist es *
STERNXSTR=s/\*/x/g
#Functions
vLog()
{
echo "$1" >> "$VLOGFILE"
}
log()
{
echo "$1" >> "$NVLOGFILE"
echo "$1" >> "$VLOGFILE"
}
rename()
# moves $1 to $2 if rn is 1,
# not anymore: otherwise set variable new to $1
{
log "rn \"$1\" to \"$2\""
if test $rn -eq 1
then
mv -- "$1" "$2"
# else # not necessary since $new is not used anymore
# new="$1"
fi
}
remove()
# deletes $1 if rn is 1, otherwise only logging
{
if test -f "$1"
then
#echo "$1"
log "rm \"$1\""
if test $rn -eq 1
then
trash "$1"
fi
fi
}
usage()
# displays help message and exit
{
echo ""
echo "help page for $0"
echo "---------------------------------"
echo "this script should not be called directly but by the Wrapper script WrapperRemoveBadSymbols.sh"
echo "Usage: bash $0 [OPTIONS] DIR"
echo " $0 removes a bunch of symbols that should not be in file names in all subdirs and files of DIR:"
echo " (with lyrics) -> remove"
echo " lyrics -> remove"
echo " blank -> _"
echo " ' - ' -> '-'"
echo " ' (Apostroph) -> remove"
echo " '\"'' (Anfuerungsstriche) -> remove"
echo " ´ -> remove"
echo " \& -> _und_"
echo " * -> x"
echo " # -> _"
echo " | -> l (small L since it looks similar)"
echo " ~ -> -"
echo " : -> _"
echo " , -> _"
echo " (){}[] -> remove"
echo " € -> EUR"
echo " $ -> USD"
echo " ä, ö, ü, Ä, Ü, Ö, ß -> ae, ..."
echo " remove various accents (not all, you can add missing ones)"
echo " consecutive _ and - and _ around - -> remove"
echo ""
echo "options"
echo " -h, --help display this help"
echo " -a, --noask --no-interactive do not prompt the user to confirm renaming (standard is asking for each file)"
echo " -l, --onlylog just logs what would have been done but not doing anything"
echo " -o [file], --outputfile [file] specifies the output (log) file (standard is rn.log)"
echo " -v [file], --voutputfile [file] specifies the verbose output log file (standard rnv.log)"
echo " -rm removes files of the form AlbumArt* and Folder.jpg and Thumbs.db"
echo " -c change everything to lowercase (doesn't work for foreign letters if diacritics remain"
echo " -- finishes the list of options, next argument is taken as the dir"
#echo " -i, --image adds date taken in beginning of file name if jpg, .png, (not yet: .mp4, .mov, ...)"
echo ""
echo "error handling"
echo " if DIR is no valid directory, the scripts eith error code 1"
exit
}
folderloc()
# returns the argument as absolute path
# arg1: if arg1 starts with /, return arg1, otherwise return $(pwd)/arg1
{
case "$1" in
/* ) echo "$1"
;;
* ) echo "$(pwd)/$1"
;;
esac
}
# considering options
# saving all options to give them recursively
ops=""
# standard behavior: ask the user is file is to be renamed
# 0 meens not asking
ask=1
# standard behavior: actually rename
# 0 meens not renaming but just log what would have be done
rn=1
# remove album pictures
# standard behavior: not removing
rmAlbumPics=0
# if file names should be converted to lower case
# standard behavior: no
lowercase=0
# the log file for verbose logging
#VLOGFILE="$1"
VLOGFILE="$(pwd)"/../rnv.log
# the log file for short logging
#NVLOGFILE="$2"
NVLOGFILE="$(pwd)"/../rn.log
#directory to be checked
dir="$(pwd)" # standard if not specified
while [ "$1" != "" ]; do
case $1 in
-h | --help ) usage;;
-o | --outputfile ) # ops="$ops -o ../$2"
shift
NVLOGFILE="$(folderloc "$1")"
;;
-v | --voutputfile ) #ops="$ops -v ../$2"
shift
VLOGFILE="$(folderloc "$1")"
;;
-a | --noask | --no-interactive ) ops="$ops -a"
ask=0
;;
-l | --onlylog ) ops="$ops -l"
rn=0
;;
-rm ) ops="$ops -rm"
rmAlbumPics=1
;;
-c ) lowercase=1
;;
-- ) shift
dir="$(folderloc "$1")"
break
;;
* ) dir="$(folderloc "$1")"
break
;;
esac
shift # must be done in any case: the next argument is $1
done
# echo "options"
# echo "asking=$ask"
# echo "rename=$rn"
# echo "image=$image"
# echo "log file=$NVLOGFILE"
# echo "verbose log file=$VLOGFILE"
# echo "direc=$dir"
#for testing purposes:
# exit
improveNames()
# this function does the work for one folder and goes into subfolders recursively
# assumes it is already in the right place
{
vLog "start with folder: $(pwd)"
printf '.'
# delete unnecessary album pictures and other useless files
if test $rmAlbumPics -eq 1
then
for alb in AlbumArt*.jpg
do
remove "$alb"
done
remove Folder.jpg
remove Thumbs.db
remove desktop.ini
fi
for fil in ./* # ../{.[^.],..?,}*
# that is: .NOTDOTanything &
# ..anything but something &
# something, not starting with dot
# also working on hidden files is not good, e.g. there are .git and many config files
do
# * als einziger Dateiname kommt auch bei leerem Ordner vor
if test -e "$fil" # file to be moved exists
then
:
# do nothing
else
continue # continue with next file
fi
# with lyrics entfernen
new="$(echo "$fil" | sed 's/(with lyrics)//g')"
# lyrics entfernen
new="$(echo "$new" | sed 's/lyrics//g')"
# Leerzeichen entfernen:
new="$(echo "$new" | sed 's/ /_/g')"
# Apostroph entfernen:
new="$(echo "$new" | sed $STRICHSTR)"
# anderes Apostroph entfernen:
new="$(echo "$new" | sed $ANDSTRSTR)"
# Anfuerungstriche entfernen:
new="$(echo "$new" | sed $ANFUERSTR)"
# wieder anderes Apostroph entfernen:
new="$(echo "$new" | sed 's/´//g')"
# Ambersands (&) entfernen:
new="$(echo "$new" | sed $AMBERSSTR)"
# Sternchen (*) entfernen:
new="$(echo "$new" | sed $STERNXSTR)"
# Rauten als Teil von Csharp:
new="$(echo "$new" | sed 's/C#/C_sharp/g')"
new="$(echo "$new" | sed 's/c#/c_sharp/g')"
# Rauten entfernen:
new="$(echo "$new" | sed 's/#/_/g')"
# vertical bars |
new="$(echo "$new" | sed 's/|/l/g')"
# comma, entfernen
new="$(echo "$new" | sed 's/,/_/g')"
# parenthesis
new="$(echo "$new" | sed 's/{//g')"
new="$(echo "$new" | sed 's/}//g')"
new="$(echo "$new" | sed 's/(//g')"
new="$(echo "$new" | sed 's/)//g')"
# [] need escaping since they are part of regexps
new="$(echo "$new" | sed 's/\[//g')"
new="$(echo "$new" | sed 's/\]//g')"
# Tilde ~
new="$(echo "$new" | sed 's/~/-/g')"
# :
new="$(echo "$new" | sed 's/:/_/g')"
# diacritics
# y as sed command says replace one character with the corresponding
# here you might want to add letters when the need arises
while true
do
new2=$new
new="$(echo "$new" | sed 'y/ãāǎàēéěèȩêīíǐìĩïōóǒòũūúǔùǖǘǚǜşļĻķĶḩģĢḨņŅŗŖĀǍÀĒÉĚÈĪÍǏÌŌÓǑÒŪÚǓÙǕǗǙǛ/aaaaeeeeeeiiiiiioooouuuuuüüüüslLkKhgGHnNrRAAAEEEEIIIIOOOOUUUUÜÜÜÜ/')"
if test "$new2" = "$new"
then
break
fi
done
# äa
new="$(echo "$new" | sed 's/ä/ae/g')"
# Äa
new="$(echo "$new" | sed 's/Ä/Ae/g')"
# áau (icelandic)
new="$(echo "$new" | sed 's/á/au/g')"
# ÁAu (icelandic)
new="$(echo "$new" | sed 's/Á/Au/g')"
# æae
new="$(echo "$new" | sed 's/æ/ae/g')"
# ÆAe
new="$(echo "$new" | sed 's/Æ/Ae/g')"
# öa
new="$(echo "$new" | sed 's/ö/oe/g')"
# Öa
new="$(echo "$new" | sed 's/Ö/Oe/g')"
# üa
new="$(echo "$new" | sed 's/ü/ue/g')"
# Üa
new="$(echo "$new" | sed 's/Ü/Ue/g')"
# ßs
new="$(echo "$new" | sed 's/ß/ss/g')"
# ðdh
new="$(echo "$new" | sed 's/ð/dh/g')"
# ÐDh
new="$(echo "$new" | sed 's/Ð/Dh/g')"
# þth
new="$(echo "$new" | sed 's/þ/th/g')"
# ÞTh
new="$(echo "$new" | sed 's/Þ/Th/g')"
# $
new="$(echo "$new" | sed 's/\$/USD/g')"
# €
new="$(echo "$new" | sed 's/€/EUR/g')"
# different jpeg extensions:
new="$(echo "$new" | sed 's/\.JPG/.jpg/g')"
new="$(echo "$new" | sed 's/\.JPEG/.jpg/g')"
new="$(echo "$new" | sed 's/\.jpeg/.jpg/g')"
if test $lowercase -eq 1 # if should be converted to upper case
then
new="$(echo "$new" | tr '[:upper:]' '[:lower:]')"
fi
#log "before complicated: $fil -> $new"
for i in "_" "\." "-"
do
# remove _ around i
isymbostr="s/_$i/$i/g"
new="$(echo "$new" | sed $isymbostr)"
#log "step 1 $i: $new"
# _ um i entfernen:
isymbostr="s/$i""_/$i/g"
#echo $isymbostr
new="$(echo "$new" | sed $isymbostr)"
#log "step 2 $i: $new; $isymbostr"
# Remove several consecutive
while true
do
isymbostr="s/$i$i/$i/g"
new2="$(echo "$new" | sed $isymbostr)"
if [ $new2 = $new ]
then break
else
new=$new2
fi
done
#log "step 3 $i: $new"
done
if test "$new" = "." -o "$new" = ".."
then
log "dont rename bc would result in $new : $(pwd)/$fil"
elif test "$fil" = "$new"
then
vLog "dont rename bc no change: $fil"
elif [[ "$fil" == ./__*__.py || "$fil" == ./__*__.html || "$fil" == ./__*__.txt ]]
then
vLog "dont rename bc probably special python file: $fil"
elif [[ "$fil" == *.class ]]
then
# sollten unproblematisch sein, beinhalten aber $s
vLog "dont rename bc probably special java class file: $fil"
# existiert eine Datei mit dem neuen Namen bereits?
elif test -e "$new" # bisher: -f (file ex. and is regular file but folders are problematic as well?!)
then
log "a file with name \"$new\" already exists, -> dont rename: $(pwd)/$fil"
elif test $ask -eq 0
then
rename "$fil" "$new"
else
# Abfrage ob man wirklich umbenennen will
echo "rename $fil to $new? enter, y, yes = yes"
read q
if [ "$q" = "" -o "$q" = "y" -o "$q" = "yes" ]
then
rename "$fil" "$new"
fi
fi
done
# renaming done, now iterate over new names
for dir in ./*; do #./{.[^.],..?,}*; do
if test -d "$dir"
then
cd "$dir"
improveNames # arbeite im Unterordner
cd ..
vLog "went back to folder $(pwd)"
fi
done
}
#actual script
# check if the argument is correct:
if [ ! -d "$dir" ]
then
log "exiting bc there is no such dir"
#echo "exiting bc there is no such dir"
exit 1
fi
# vLog "move to $dir"
cd "$dir"
# reset log files
echo "" > "$NVLOGFILE"
echo "" > "$VLOGFILE"
improveNames
# merke: in Bedingungsklammern [] von if muss am Anfang und am Ende ein " " (blank) stehen!!
# merke: Hinweise zu Bedingungen in if abrufbar unter "man test"
# merke: bei Zuweisungen mit = darf zwischen Variablenname und = kein Leerzweichen sein!
# (sonst wird Variablenname als command interpretiert)