Attempt to set up application

README.md

@@ -1,13 +1,12 @@
 ---
-title: Dutch Youth Comment Classifier
-emoji: 📉
-colorFrom: blue
-colorTo: gray
+title: Comment Classifier
+emoji: ☑️
+colorFrom: pink
+colorTo: blue
 sdk: gradio
-sdk_version: 3.33.1
+sdk_version: 3.28.1
 app_file: app.py
 pinned: false
-license: cc-by-nc-nd-4.0
 ---
 
 Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,71 @@
+import gradio as gr
+from huggingface_hub import from_pretrained_fastai
+
+from src.start_up.start_up_rbs import create_weak_rbs, create_strong_rbs
+from src.start_up.start_up_gibberish import create_gibberish_detector
+
+# Start up modules
+#   Initiate model
+learner = from_pretrained_fastai("felixolivier/comment-classifier")  # TODO rename this
+verdict_map = {
+    0: 'Inappropriate',
+    1: 'Allowed'
+}
+
+#   Initiate rule based systems (RBS)
+weak_rbs = create_weak_rbs()
+strong_rbs = create_strong_rbs()
+
+#   Initiate gibberish detector
+gibberish_detector = create_gibberish_detector()
+
+
+# Define function to judge comment
+def predict(input_text):
+
+    # Get model predictions
+    predictions = learner.predict(input_text)
+
+    # Pass through weak RBS
+    allows_weak, reasons_weak, highlights_weak = weak_rbs.allows(input_text)
+
+    # Pass through strong RBS
+    allows_strong, reasons_strong, highlights_strong = strong_rbs.allows(input_text)
+
+    # Pass through gibberish detector
+    is_gibberish_free = gibberish_detector.predict(input_text)
+
+    # Construct json respons
+    to_return = {
+        'model': {
+            'allowed': bool(int(predictions[0])),
+            'verdict': verdict_map[int(predictions[0])],
+            'highlights': [],
+            'reasons': ['Machine learning model does not approve' for el in [predictions[0]] if not int(el)]
+        },
+        'gibberish': {
+            'allowed': bool(is_gibberish_free),
+            'verdict': verdict_map[int(is_gibberish_free)],
+            'highlights': [],
+            'reasons': ['Comment text contains gibberish' for el in [is_gibberish_free] if not el]
+        },
+        'weak_rbs': {
+            'allowed': allows_weak,
+            'verdict': verdict_map[allows_weak],
+            'highlights': highlights_weak,
+            'reasons': reasons_weak
+        },
+        'strong_rbs': {
+            'allowed': allows_strong,
+            'verdict': verdict_map[allows_strong],
+            'highlights': highlights_strong,
+            'reasons': reasons_strong
+        },
+    }
+
+    return to_return
+
+
+# Set up app
+iface = gr.Interface(fn=predict, inputs="text", outputs="text")
+iface.launch(share=False)

requirements.txt

@@ -0,0 +1,9 @@
+transformers
+torch
+fastai
+fastcore
+toml
+gibberish-detector
+pandas
+urlextract
+bs4

src/config.py

@@ -0,0 +1,5 @@
+config = {
+    'gibberish_model': 'src/gibberish_detection/approved_comments.model',
+    'bad_words_strict': 'src/rule_based_system/word_lists/strictly_bad_words.csv',
+    'bad_words_ambiguous': 'src/rule_based_system/word_lists/ambiguous_bad_words.csv',
+}

src/gibberish_detection/GibberishDetector.py

@@ -0,0 +1,25 @@
+class GibberishDetector:
+    """
+        Detects whether the comment is actual text or a random sequence of characters.
+        The model has been trained to learn character sequence probabilities from a set of approximately 4 million
+        approved comments.
+    """
+
+    model = None
+    skip_sequences = ['duh', 'ah']  # tokens that are allowed even tough the model might see these as gibberish
+
+    def __init__(self, model):
+        self.model = model
+
+    def predict(self, text):
+        return not self.contains_gibberish(text)
+
+    def contains_gibberish(self, text):
+
+        # Skip over tokens that often are misjudged by the model
+        for skip in self.skip_sequences:
+            if skip in text:
+                return False
+
+        # Pass text through model
+        return self.model.is_gibberish(text)

src/gibberish_detection/approved_comments.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:26c786281ab9642063f9b6c0ad2b07d6122cd00b5c574aec0d7be63d3d1438f6
+size 29971

src/rule_based_system/BadWordRule.py

@@ -0,0 +1,47 @@
+from src.rule_based_system.Rule import Rule
+
+from src.rule_based_system.TextLengthRule import TEXT_SIZE_LIMIT
+from src.rule_based_system.Verdict import Verdict
+
+
+class BadWordRule(Rule):
+    """
+    Bad words obtained from corners of the internet you do not want to visit:
+        - https://www.ensie.nl/scheldwoordenboek#
+        - https://scheldwoorden.goedbegin.nl/
+        - https://nl.wiktionary.org/wiki/Categorie:Scheldwoord_in_het_Nederlands
+        - https://www.lannoo.be/sites/default/files/books/issuu/9789401453417.pdf
+        - https://www.dutchmultimedia.nl/meest-verschrikkelijke-engelse-scheldwoorden/
+        - https://www.dutchmultimedia.nl/scheldwoordenboek-1-000-den-nederlandse-scheldwoorden/
+        - https://www.henkyspapiamento.com/10-papiaments-scheldwoorden-die-we-liever-niet-horen/
+        - https://volkabulaire.nl/tag/scheldwoorden/
+        - https://data.world/wordlists/dirty-naughty-obscene-and-otherwise-bad-words-in-dutch
+    """
+
+    bad_words = None
+
+    def __init__(self,  bad_words: list, strict: bool):
+        self.bad_words = bad_words
+        self.strict = strict
+
+    def get_verdict(self, comment_text: str) -> Verdict:
+        comment_text = comment_text[0:TEXT_SIZE_LIMIT]
+
+        bad_words = self.find_bad_words(comment_text.split())
+
+        return Verdict(len(bad_words) == 0, bad_words)
+
+    def find_bad_words(self, text: list) -> list:
+        detected_bad_words = []
+        for word in text:
+            if word in self.bad_words:
+                detected_bad_words.append(word)
+
+        return detected_bad_words
+
+    def is_strict(self) -> bool:
+        return self.strict
+
+    def get_rule_description(self) -> str:
+        return "Comment text contained %s inappropriate words" % \
+               ('strictly' if self.is_strict() else 'ambiguous')

src/rule_based_system/HTMLRule.py

@@ -0,0 +1,27 @@
+from bs4 import BeautifulSoup
+
+from src.rule_based_system.Rule import Rule
+from src.rule_based_system.Verdict import Verdict
+
+
+class HTMLRule(Rule):
+
+    def get_verdict(self, comment_text: str) -> Verdict:
+        html = self.find_html(comment_text)
+
+        return Verdict(len(html) == 0, html)
+
+    @staticmethod
+    def find_html(text: str) -> list:
+        html = BeautifulSoup(text, "html.parser").find_all()
+
+        return [str(tag) for tag in html]
+
+    def is_strict(self) -> bool:
+        """
+        This rule occasionally removes appropriate comments when names are enclosed in triangular brackets e.g. <name>
+        """
+        return False
+
+    def get_rule_description(self) -> str:
+        return 'HTML used in comment text'

src/rule_based_system/LanguageRule.py

@@ -0,0 +1,19 @@
+from src.rule_based_system.Rule import Rule
+
+
+class LanguageRule(Rule):
+
+    def get_verdict(self, comment_text: str) -> bool:
+        """
+            This rule requires further research. The library "langdetect" allows for language detection,
+            but when applied to jeugdjournaal comments returns unstable results.
+            A simpler approach could be by checking the percentage of non latin characters,
+            but not taking into account smileys and punctuation
+        """
+        raise NotImplementedError()
+
+    def is_strict(self) -> bool:
+        return False
+
+
+language_rule = LanguageRule()

src/rule_based_system/PersonalDetailsRule.py

@@ -0,0 +1,34 @@
+import re
+
+from src.rule_based_system.Rule import Rule
+from src.rule_based_system.TextLengthRule import TEXT_SIZE_LIMIT
+from src.rule_based_system.Verdict import Verdict
+
+
+class PersonalDetailsRule(Rule):
+
+    def __init__(self, regexes: list, strict: bool):
+        self.regexes = regexes
+        self.strict = strict
+
+    def get_verdict(self, comment_text: str) -> Verdict:
+        comment_text = comment_text[0:TEXT_SIZE_LIMIT]
+
+        personal_details = self.find_personal_details(comment_text)
+
+        return Verdict(len(personal_details) == 0, personal_details)
+
+    def find_personal_details(self, text: str) -> list:
+        details = []
+        for regex in self.regexes:
+            matches = re.findall(regex, text)
+            details += matches
+
+        return details
+
+    def is_strict(self) -> bool:
+        return self.strict
+
+    @staticmethod
+    def get_rule_description() -> str:
+        return 'Personal details were mentioned in text'

src/rule_based_system/Rule.py

@@ -0,0 +1,25 @@
+from abc import ABC
+
+from src.rule_based_system.Verdict import Verdict
+
+
+class Rule(ABC):
+
+    def get_verdict(self, comment_text: str) -> Verdict:
+        """
+            Takes the comment text as input, tests a specific rule and returns a verdict,
+            which contains whether the comment is allowed according to the specific rule and
+            contains a list of substrings in the comment that may explain why a comment was
+            marked as inappropriate.
+        """
+        pass
+
+    def is_strict(self) -> bool:
+        """
+            Returns True if rule can be used directly. False if results may be ambiguous.
+        """
+        pass
+
+    @staticmethod
+    def get_rule_description() -> str:
+        pass

src/rule_based_system/RuleBasedSystem.py

@@ -0,0 +1,17 @@
+class RuleBasedSystem:
+    rules = []
+
+    def __init__(self, rules: list):
+        self.rules = rules
+
+    def allows(self, comment_text: str) -> (bool, list):
+        allows, reasons, highlights = True, [], []
+
+        for rule in self.rules:
+            verdict = rule.get_verdict(comment_text)
+            if not verdict.allowed:
+                allows = False
+                reasons.append(rule.get_rule_description())
+                highlights += verdict.highlights
+
+        return allows, reasons, highlights

src/rule_based_system/TextLengthRule.py

@@ -0,0 +1,29 @@
+from src.rule_based_system.Rule import Rule
+from src.rule_based_system.Verdict import Verdict
+
+TEXT_SIZE_LIMIT = 500
+
+
+class TextLengthRule(Rule):
+
+    def get_verdict(self, comment_text: str) -> Verdict:
+        allows = True \
+            and not self.is_empty(comment_text) \
+            and not self.is_too_long(comment_text, TEXT_SIZE_LIMIT)
+
+        return Verdict(allows, [])
+
+    @staticmethod
+    def is_empty(text):
+        return len(text) == 0
+
+    @staticmethod
+    def is_too_long(text, limit):
+        return len(text) > limit
+
+    def is_strict(self) -> bool:
+        return True
+
+    @staticmethod
+    def get_rule_description() -> str:
+        return 'Inappropriate text length'

src/rule_based_system/UrlRule.py

@@ -0,0 +1,31 @@
+from src.rule_based_system.Rule import Rule
+from src.rule_based_system.Verdict import Verdict
+
+
+class UrlRule(Rule):
+
+    url_extractor = None
+
+    def __init__(self, url_extractor):
+        self.url_extractor = url_extractor
+
+    def get_verdict(self, comment_text: str) -> Verdict:
+        urls = self.find_urls(comment_text)
+
+        return Verdict(len(urls) == 0, urls)
+
+    def find_urls(self, text: str) -> list:
+        urls = self.url_extractor.find_urls(text)
+
+        # url_extractor does not find url with spaces. Add extra check for urls like http:// goatse info
+        if len(urls) == 0 and 'http' in text:
+            urls = ['http']
+
+        return urls
+
+    def is_strict(self) -> bool:
+        return True
+
+    @staticmethod
+    def get_rule_description() -> str:
+        return 'Url was mentioned in text'

src/rule_based_system/Verdict.py

@@ -0,0 +1,5 @@
+class Verdict:
+
+    def __init__(self, allowed: bool, highlights: list = []):
+        self.allowed = allowed
+        self.highlights = highlights

src/rule_based_system/word_lists/ambiguous_bad_words.csv

@@ -0,0 +1,1467 @@
+parasiet
+varkenskop
+halvezool
+pannekoek
+makak
+dameskapper
+pennelikker
+papomslag
+lafaard
+takketrol
+lolo
+flaps
+steenezel
+aalskakker
+aarselaar
+ass
+gamefreak
+afzeiken
+zwijn
+kechba
+yoghurtventje
+homo
+tollie
+knor
+koolgalmug
+dike
+zandloper
+cunnie
+anita
+vot
+klootoog
+masque
+railnicht
+zalfpot
+zeiksnor
+kapsoneslijer
+kaffer
+zeurkous
+pasjakroet
+piemel
+boerenhufter
+baarmoederslijmvlieg
+bedrijfspoedel
+fatsich
+kouloh
+wankjob
+kaketoe
+kelerelijer
+lijpo
+achterbuurtfossiel
+camslet
+gangster
+kloefkapper
+fuckwad
+kankernicht
+schaamluis
+geitenbreier
+muilezelin
+flappie
+choad
+perenplukker
+uiersnuiver
+cockshit
+aarslander
+parkschijter
+kuthoer
+onderkruipsel
+apenkont
+zaadbal
+aarsvis
+douche-fag
+olifantenlul
+prod
+aapmens
+optyfen
+makaak
+piggelmee
+sodemieter
+naadschurftsmurfigebeftekkel
+prostituee
+tampony
+dumbfuck
+mofo
+droplul
+apenhaar
+dakhaas
+parkinghomo
+mafkaas
+twats
+zeiker
+rapenschijter
+zaadzwerver
+dickjuice
+paardenplasserwasser
+karhengst
+inktfis
+mafketel
+assbanger
+okkenootje
+anita
+neetoor
+eierzak
+jerkass
+geitenneuker
+papegaai
+snikkel
+aarsmaad
+afberen
+takkentrut
+dakduif
+badaap
+muts
+autist
+puto
+raaskallen
+schaamliplikker
+baarvader
+faggotcock
+poontang
+cuntlicker
+koffieboon
+padoog
+vuilak
+drooggeiler
+fag
+beurt
+kakkie
+afrossen
+pekelteef
+olalul
+loser
+kloothannes
+augurk
+fuckhole
+avocadoneuker
+kooch
+panooch
+piemelhoofdje
+lapzwans
+fielt
+graftak
+idioot
+kurwa
+whore
+fuckbag
+paardenlul
+schadewijf
+gladjakker
+bep
+cockfucker
+broekhoesten
+kriel
+cockmuncher
+gladiool
+schijtlijster
+lomperd
+loeder
+pekelhoer
+sanka
+sekreet
+dweil
+oesterkut
+pothoer
+skeet
+uit-de-baarmoeder-gerukte-tien-tepelige-homofiel
+fuckin
+clit
+piss
+aarsklodder
+natkut
+leipo
+boemelen
+facadeklasher
+harpij
+jam-bek
+paardebeffer
+aardgeest
+reetkever
+aap
+rugridder
+cock
+eiermix-imitatie
+haai
+muffdiver
+paardenanus
+campinghoer
+kechies
+vetzak
+assgoblin
+bedonderen
+knob
+jetslet
+woeshoem
+homodumbshit
+zalf
+boekenwurm
+asscracker
+minger
+ini-mini-scheefgepoepte-pornokabouter
+halvegare
+zeikstraal
+gook
+ruski
+yogoteef
+zaadvarken
+sodomiet
+retenlikker
+aambeienlikker
+kakbal
+kutstreek
+caveman
+oester
+tit
+achterpoter
+bellend
+heeb
+assbag
+smeerpijp
+janettescheter
+ragkut
+zakrat
+bouwdoos
+gaylord
+pielemans
+peul
+zemmer
+bloodclaat
+liefdesgrot
+ass-pirate
+dookie
+penislijder
+quasimodo
+slettenbak
+muff
+penisfucker
+anustoerist
+fucknutt
+neukpaal
+pijphomo
+bumblefuck
+asshole
+pruim
+heihaas
+vaginavochtabsorbeerder
+schatje
+peckerhead
+cunt
+piggel
+luibuis
+anussap-verzuring
+zenuwlijer
+mothafuckin'
+paal
+hell
+shitter
+billelikkertje
+mafkees
+flamoes
+parkiet
+uienreet
+achterlader
+bitchtits
+honkey
+splooge
+slag
+adderengebroedsels
+janksnor
+fucktard
+fucktart
+rotmof
+insekt
+kringspiermusketier
+cockburger
+shitbag
+naarling
+luibak
+cooter
+sod-off
+racechinees
+bruinwerker
+racesjinees
+gaybob
+superkut
+hondenlul
+klapkut
+foefkop
+lijer
+pigmentvreter
+stinker
+paarsedruiveneikel
+fuckup
+baanduivel
+lauwbikker
+gay
+dicksucking
+hakker
+vag
+kontneuker
+nul
+kruimelbuik
+assface
+oen
+junglebunny
+aaprikaan
+anuspiloot
+landloper
+punani
+pijpbekkieg
+lameass
+penis
+tollielikker
+dònchin
+pampa
+huisdealer
+pierlala
+zwijnjak
+pepermuntvreter
+gadoodengeefmijbrood
+viespeuk
+anuspiraat
+pijpert
+pijpmonk
+egocentrischebullebak
+ezel
+kenau
+vergallen
+assmunch
+zeikhannes
+queer
+arsehole
+yid
+raas
+infanterist
+dickmilk
+tamponkut
+ettertje
+mcfagget
+labrat
+picorniekop
+darmgas-wildebras
+wabipi
+cumtart
+acrobaat
+centenschraper
+aarsbaard
+autokraat
+suckass
+nigga
+choco
+tang
+pijppiloot
+beaner
+morsebel
+takketut
+dirkdoos
+bastard
+faggit
+modderduivel
+asbestmuis
+damhoer
+cuntass
+oksel
+passoa-slet
+kakkerlak
+zeekbos
+mick
+edammer
+darmkikker
+pezewever
+zibi
+plucheplakker
+nachtuil
+xylofoonneuker
+lampenhoer
+anusreflector
+fuckhead
+billig
+wippen
+poonani
+politiemuts
+schuurmeid
+naadhopper
+jap
+palenlaaier
+dwaas
+spuiten
+renob
+pokkenwijf
+lampelul
+naaiaap
+dreuzel
+aardpiraat
+doshi
+arrefakker
+schaamhaarverzamelaar
+piemelbrie
+randdebiel
+lesbo
+chocoladesnol
+kechje
+testicle
+nitwit
+reetroeier
+kadaver
+shitbrains
+asslicker
+crapuul
+vampier
+bitchboy
+baarmoederscheefgepoept-kuttekind
+goddamn
+zaadvragende-budcontainer
+asswipe
+gayfuck
+olifantebabyface
+standje
+fucking
+dumbass
+kankerhond
+slapper
+maf
+poot
+rammelhoofd
+maangezicht
+reutelen
+kakboer
+opzouten
+paardenpijper
+cumbubble
+douchebag
+temeier
+boerenkaffer
+cockmonkey
+teef
+slempen
+zaadhamsteraar
+bokkenrijder
+aambeienschoffelaar
+pushi
+deggo
+brilsmurf
+anuspuntverknetteraar
+asscock
+piemelgymmer
+dickface
+fuckbutt
+penisspuiter
+appelzwijn
+pensklepper
+schlemiel
+badmuts
+slijmerd
+asshat
+aarsridder
+schelm
+hoerenjong
+schurftkop
+keilef
+punanny
+moederneuker
+zakkendroller
+bermslet
+smeg
+lauwwaterdrinker
+zaadspuitende
+ragnicht
+zaadcontainer
+tietvlieg
+engerd
+zaadslikker
+achterlijke
+shittiest
+meretrix
+papzak
+aarsbeffer
+ectoplasma
+paardenpenislikker
+kaashoer
+nageboorte
+reetveger
+queerhole
+pokkenlijder
+makako
+scrote
+darmcoureur
+gajeskop
+apekop
+ossenkop
+hondenneuker
+nasivreter
+lawaaipapegaai
+bezem
+flapkut
+asshopper
+djuka
+shitass
+jp
+uil
+zaadschieter
+knijpkont
+kreng
+zandhaas
+zenuwenlijer
+lesbian
+ballen
+asshead
+hol
+aarsketser
+schurk
+inktviskut
+bonk
+indoending
+wanker
+goddam
+snitch
+huppelkut
+coronalijer
+kike
+naaien
+buttfucka
+zaadvrager
+dòlò
+kuttenkop
+hamerhomo
+vaginalehangflapper
+haaibaai
+munging
+dickhead
+klafte
+netenvreter
+kaashaas
+rampenbek
+prick
+befborstel
+labbekak
+vjayjay
+manwijf
+dumbshit
+apengaper
+aardworst
+dickwad
+boeler
+peniskokerlikker
+klapluis
+clitfuck
+chink
+kippenneuker
+klier
+mafkikker
+konthond
+asslick
+spintmijt
+bugger
+tonto
+palingstekker
+gaytard
+afrotten
+aalscholver
+phenomeen
+ouwehoeren
+twatlips
+opsodemieteren
+cocknugget
+kaapstander
+schaamhaaretendebaffer
+schijtlaars
+bille-likkers
+nutsack
+piaprisma
+zakbreuk
+kloothommel
+janlul
+jizz
+fuckersucker
+triootje
+aarsatleet
+shitface
+kluns
+bollox
+buffelen
+kontol
+krielkip
+familycocktail
+babaloekoe
+piemelwijf
+ito
+roetmop
+assbite
+fucked
+bhenchod
+flikken
+walvisganger
+fabricagefout
+botterik
+droogkloot
+aso-big
+cocksmith
+wipkip
+pindachinees
+rekel
+darmstad
+sukkel
+populist
+verdomme
+beheime
+pijpen
+shitty
+nazi
+kutvent
+hamel
+potver
+wuftje
+achterbaksestoephoer
+papist
+rampdebiel
+dickfucker
+fuckwit
+pleurislijder
+kakbroek
+sandnigger
+penispijper
+perebek
+smeerkanis
+apekloot
+lantarenslet
+lomperik
+clitface
+kapoen
+pestboer
+boerenkinkel
+uit-de-baarmoeder-geruktedeurknop
+watervlo
+tittyfuck
+anuspapegaai
+johny
+lezzie
+miljaar
+vlerk
+ass-hat
+palen
+proleet
+zeurpiet
+yvesmoeder
+hersenlijer
+balls
+tyfuslijder
+whorebag
+pedocyclomasturbant
+zaadsnuiver
+fascist
+arroganteklitzuigende-wrattenhoofd
+shitcanned
+afrukken
+grobbejanus
+zakkenvuller
+kettingzeugg
+fagbag
+baarmoeder
+daaps
+naadje
+dicksucker
+remsporen
+pollock
+schaap
+zaksel
+toeter
+kakmadam
+holtor
+dronkenlap
+maaghond
+lamzak
+zeikbeer
+aarsbanaan
+stinkbok
+djoeka
+fucker
+zaadheks
+paki
+aftrekken
+sambalburger
+satan
+konjo
+faggot
+zeiken
+raamneger
+kraut
+shitting
+maaghoer
+paardenpenis
+palingkop
+bekken
+biljartbal
+breezerslet
+fellatio
+cunnilingus
+appelflap
+cow
+aartschurk
+autocraat
+pietlut
+stomkop
+pussies
+shitbagger
+gash
+dago
+fuckoff
+boner
+motherfucking
+matennaaier
+puistenkop
+tangelteef
+clunge
+klootviool
+quasimodom
+assclown
+naaimachine
+trien
+pantoffelneuker
+wratzwijn
+cumjockey
+axwound
+inktlul
+éénoog
+engerling
+hoerenkind
+piemelpatser
+shiz
+rapalje
+geit
+specht
+darmtoerist
+hapsnurker
+mispunt
+polesmoker
+kinkel
+hoerenjager
+cuntrag
+spleetoog
+aambeienbeffer
+baby
+carnavals-inca
+kaaskop
+wafelijzer
+zaadnaad
+lekkerpieper
+sabberkut
+chesticle
+pielesnuiver
+okselharenbillereetzweetzuur
+schietmot
+nonce
+assshole
+fuckbutter
+bampot
+lafbek
+zaadhoer
+assnigger
+pinda
+piek
+assfuck
+gesodemieter
+vlooienzak
+latexnicht
+nigaboo
+tits
+tard
+zeikerd
+guido
+egoist
+noppeshoer
+cockknoker
+gluiperd
+nageboortegezwe
+jandoedel
+cocksmoker
+bolle
+wannabee
+taboetstabiske
+aardworm
+wank
+aarsneger
+kankerlijer
+ganzeboard
+uit-de-baarmoeder-gerukte-tien-tepelige-kut-trut
+passoatrut
+cocksniffer
+carnavalskaper
+campusnerd
+babypoedersnuiver
+totebel
+vleesroos
+penishoofd
+schobbejak
+wandelende-reklame-voor-kachelglans
+paardelullo
+aarsgarnaal
+poon
+carpetmuncher
+anus
+lelijkerd
+coon
+git
+assshit
+fanny
+kakteef
+badgast
+flauwerd
+lamagezicht
+hockeytut
+cockmongler
+oelewap
+parel
+dickweed
+patagonier
+kankerlijder
+buttfucker
+barslet
+watje
+dickweasel
+lèrchi
+reet
+dòndòn
+mothafucka
+cazzo
+besodemieteren
+geteisem
+nanoneuker
+klojo
+olifantenschimmelslurf
+flessentrekker
+strontzak
+karonje
+poep
+zakcel
+zwakbegaafde
+fuckass
+hoempert
+zaadstaafteler
+kontos
+coochie
+kontkruiper
+armoedverspreider
+shiznit
+aarsworm
+pandapikkenpijper
+deuggleuf
+tapijtnek
+waailap
+cockjockey
+piezer
+cumslut
+fietspompenlullegezicht
+azteek
+kechie
+soepkip
+barg
+aarsvijand
+schijtluis
+cockhead
+sapcentrifuge
+apenkut
+prij
+smeerlap
+heks
+dipshit
+mikrodebiel
+baarmoeder-discokip
+heaumeau
+truthola
+mietje
+asfaltridder
+cumdumpster
+aarslikker
+klotenklapper
+afzuigen
+eeltneus
+shag
+heikneuter
+sal
+edeldarm
+kakker
+paardenpikkenpoetser
+variétéhoer
+cockface
+koeskoesvreter
+galsmoel
+paardekut
+galkankerdebiel
+pleurislaaier
+dickhole
+imbeciel
+lamstraal
+assmuncher
+goddamnit
+leuter
+rabauter
+jigaboo
+gannef
+sakkers
+jetonslikker
+viruswappie
+fok
+eend
+anuspenetreerder
+kamerolifant
+macaronivreter
+gans
+sacoche
+bint
+boerenkarhengst
+baardaap
+gooch
+spast
+wasbakkenpisser
+zaadje
+shithouse
+secreet
+lullo
+badjufbeffer
+linkmiegel
+zeur
+ass-jabber
+pukkelbek
+sambalvreter
+wanhopigedarmrukker
+del
+incel
+maan-rovers
+klapperaap
+kamelenneuker
+zanikert
+anuspieper
+fuckbrain
+flikker
+pijpenkoplikker
+beest
+charlatan
+wetback
+bullshit
+cockwaffle
+krijslijster
+aarsaap
+langstaart
+kaalkop
+pijphoer
+zatmuil
+zeiklijster
+doos
+penisbanger
+anussabbelaar
+shit
+tampontrekker
+schijthoofd
+reetridder
+coochy
+griet
+fleer
+afwerkplaats
+aardvarken
+balhaar
+gambaslet
+spook
+pik
+pummel
+assjacker
+daaglaker
+uilskuiken
+luiskop
+jeannette
+waterhoofd
+dagblinde
+zoubi
+zemel
+fuckboy
+shitcunt
+pijpdeuramateur
+hangjas
+naakt
+aansteller
+snoever
+twatwaffle
+vaginavarken
+patser
+douche
+cretin
+kyke
+darmneuker
+hond
+pierewipper
+kwakzalver
+giechelkut
+feltch
+taig
+nondeju
+krentekakker
+piesen
+takenboeker
+vetklep
+wrattenzwijn
+nietsnut
+pijpelullebakkes
+lamlul
+galgenbrok
+jagoff
+mierenneuker
+kaasrandknabbelaar
+pijpmond
+graftyfuskankerpleuris
+teringlijder
+aarsmade
+hode
+godver
+kantoorpik
+piemelboxer
+mosselhoer
+kakmaker
+kontengezicht
+zwartjoekel
+cracker
+apache
+okselschimmelhoer
+pias
+piel
+klere
+capsonestrut
+zeekbakkes
+shithole
+publiciteitsgeil
+debiel
+judas
+mispruim
+kanker
+hangbuikvarkenkonteneuker
+zakkenwasser
+lardass
+godverdomme
+kloot
+flapdrol
+maajen
+assbandit
+asssucker
+baccilledrager
+shitdick
+fucknut
+asperge
+gat
+anusuitscheiding
+beaver
+cuntface
+shitstain
+darmafval
+zapoteek
+draaikont
+apenbatser
+baffer
+hamsterpampe
+chinc
+ruigpoot
+aardappelluis
+spic
+schijterd
+cocksucker
+pantyneuker
+titfuck
+dickwod
+motherfucker
+pedopenis
+niglet
+waterdrinker
+tering
+quikteef
+humping
+brotherfucker
+pampaman
+snatch
+zaadstengel
+shitspitter
+penispuffer
+oelewapper
+galbak
+olijfneuker
+butt-pirate
+hondsvot
+aarsfreter
+pekelharing
+naaidoos
+zaadvreter
+pendeu
+aarshaarluis
+farmer
+vandaal
+tongzoeng
+dickmonger
+okselhaas
+uitgekotstekamelenkut
+hondenkut
+ouwehoedendoos
+cunthole
+pazop
+clusterfuck
+hoerenzoon
+camperhoer
+palurk
+quartrozo
+chode
+aardpeer
+pussy
+klaplul
+munter
+pagadder
+penozeg
+pindapoepchinees
+crap
+aars
+fatass
+asswad
+bengel
+paashaasschaamhaarverzamelaar
+patin
+carnavals-mussolini
+polak
+flamer
+neger
+hamsterpijper
+gringo
+etterbak
+autodidact
+slutbag
+zaadslet
+flathoer
+pissed
+arse
+takkehoer
+keutelkut
+lambak
+rapey
+aarskind
+haas
+porchmonkey
+jambekslikker
+dumshit
+satéboer
+campingslet
+bosneuker
+hamsteraar
+aardappelhoer
+bukkake
+merde
+kokosmakroon
+pijpslet
+nicht
+covidioot
+tyfus
+gedoogzone
+tokkie
+nigger
+belazeren
+dickbag
+pijpgeit
+dickslap
+bitchass
+kontelikker
+bloody
+pijpbek
+stootje
+taart
+kaasknobbel
+bangerd
+pierlalla
+zandvlo
+uit-de-baarmoeder-gerukte-mislukte-kuttekop
+palmboomneuker
+cum
+cuntslut
+moffenhoer
+schaamlipgezwel
+troelala
+oppleuren
+doochbag
+kutlul
+braadvarken
+klooien
+hockeytrut
+snotolf
+sodeflikker
+asspirate
+kroeskop
+zattekop
+cockmongruel
+lampekap
+wasknijper
+zaagselkop
+shitfaced
+douchewaffle
+lavabosmoel
+paardenneuker
+aarshaarverzamelaar
+aambeihaarwasser
+vullis
+smiecht
+asielzoeker
+gratenbaal
+rimjob
+boerenheikneuter
+gabber
+gluiper
+apenjong
+apenwaffel
+ossenlul
+pedo
+pecker
+qwarf
+parkeerkut
+wop
+zuiplap
+greppeldel
+tart
+sodeju
+kruiper
+ramenlikker
+huzarenhoop
+eikellikker
+aarslog
+klotenbijter
+galgengebroed
+pakul
+dicktickler
+satraap
+tankreet
+palingvel
+anusridder
+baggerduiker
+matje
+lap
+snotaap
+babok
+ebverzuiper
+hangbuikzwijn
+uit-de-baarmoeder-gerukte-tien-tepelige-koeie-stront
+truus
+etter
+assfucker
+gaywad
+kankerhoer
+boomer
+aarshaarknabbelaar
+bloedlijer
+damn
+eendekweker
+lantaarnpaalhoer
+pissflaps
+piemelaar
+zemmel
+cumguzzler
+fagtard
+varken
+dumass
+batti
+dick-sneeze
+cocksmoke
+gayfuckist
+paplap
+aardbanaan
+raketpiraat
+stinkerd
+vagina
+yoghurtpisser
+miegels
+geilneef
+baarmoederbuikwandschimmel
+makreel
+cockass
+kaasnicht
+schimmelkut
+negro
+cockmaster
+ho
+kootch
+shithead
+fucks
+penisworstelaar
+twat
+sloerie
+jenevertrut
+patjepeeër
+abortuskind
+dickbeaters
+schlong
+augurkenneuker
+schaapskop
+ploert
+klooi
+welzijnsmafia
+asbak
+assmonkey
+drol
+druiloor
+cijferneuker
+gluipsnor
+shitbreath
+kwakkieg
+fuckwitt
+bollocks
+skank
+fuckface
+peniskoker
+cockbite
+cocknose
+gratenkut
+campesneurdo
+pijpzuiger
+schaamlikkerkip
+plurk
+schapenneuker
+addergebroed
+zeurzak
+moffrika
+kanen
+rukken
+piemelpiraat
+aso
+mariku
+zak
+cretino
+satehoer
+kont
+pommel
+pedopyrofilatomaan
+taalnazi
+eikel
+absjaar
+ginger
+internetverslaafde-neuker
+ponylul
+pot
+eendenkont
+koe
+poepen
+patjakker
+kleumer
+koño
+zuigen
+zadelruiker
+offroadie
+apenrukker
+schweinhund
+niggers
+klaarkomen
+hammehapper
+augurkenturk
+zwartjan
+skullfuck
+bruisend
+sikbille
+kebber
+anusvreter
+handvat
+quist
+dickfuck
+paardebrei
+pedomeester
+spaghettivreter
+mapangpang
+chocoladehoer
+gadverdamme
+babbalipviskikker
+klote
+oelepoeper
+minge
+aarstulp
+ransaap
+slut
+bitchy
+tampontoerist
+graftakketeef
+penislikker
+queef
+dyke
+oezo
+paardelid
+tiefus
+krimper
+klerebeer
+etterbuil
+jood
+spick
+handjob
+malaka
+paardenpikkerslikker
+aapjesviller
+papagaaiekuttenbefkonijn
+aarsbaars
+pakhuisrat
+bitches
+flamingo
+aarshaarfohner
+armoedzaaier
+schaampik
+mafklapper
+piemelteef
+piemelpunt
+gayass
+flatslet
+maffer
+belhamel
+vajayjay
+capsoneslijer
+pestbek
+eikelkluiver
+tifus
+whoreface
+potverdorie
+ouwehoer
+yoyolul
+bulderbast
+potverdikkeme
+jackass
+broodaap
+raceplee
+poonany
+schuinsmarcheerder
+pestlijder
+dildo
+aasgier
+kak
+zeekmem
+apenflikkerneuker
+fileklever
+peukenneuker
+kwal
+cholera
+kontonbonker
+darmfloraflirter
+nakko
+vingeren
+bleekscheet
+papabaf
+dicks
+vaselinevriend
+dick
+klootjesvolk
+varkenshol
+peniszuiger
+cabaretlul
+adder
+pedomaan
+eendeneikel
+klerelijer
+va-j-j
+brugpieper
+blowjob
+invertebraat
+unclefucker
+zuurpruim
+punta
+sloef
+gaydo
+smous
+fudgepacker
+bosneger
+asses
+kakhuis
+parelridder
+vaking
+downie
+dikke
+dikzak

src/rule_based_system/word_lists/strictly_bad_words.csv

@@ -0,0 +1,100 @@
+klootzak
+hoer
+bitch
+slet
+trut
+kutwijf
+mongool
+aflebberen
+beffen
+bef
+bokkelul
+botergeil
+boerelul
+dombo
+boerenpummel
+godverdamme
+hoer
+hoerenbuurt
+hoerenloper
+hoerig
+hufter
+klootzak
+kloten
+kontneuken
+kut
+kuttelikkertje
+lul
+lul-de-behanger
+lulhannes
+lummel
+mof
+neuken
+neukstier
+oetlul
+opgeilen
+opkankeren
+oprotten
+paardelul
+portiekslet
+rothoer
+rotzak
+rukhond
+schijt
+schijten
+schoft
+slet
+sletterig
+slik mijn zaad
+snol
+standje-69
+stoephoer
+stront
+sufferd
+teringlijer
+trottoir prostituée
+trottoirteef
+verkloten
+verneuken
+wijf
+fuck
+fuk
+fack
+fak
+fock
+afgebefte del
+prickteaser
+aarshaar
+paardekloot
+thundercunt
+aarsbeer
+feck
+quikhoer
+galgenaas
+bafaap
+fagfucker
+pussylicking
+apeneuker
+internetslet
+takkenteef
+augurklul
+queerbait
+aarsslijm
+paashaasneuker
+pijpendemonkyboy
+fuckstick
+piemelpuist
+penispukkelpuist
+schaamluisbedrieger
+eikelbijter
+kech
+kahba
+dikoe
+conjo
+kaulo
+kowed
+poeta
+puta
+tabon
+uchi
+ucci

src/start_up/start_up_bad_words_rule.py

@@ -0,0 +1,12 @@
+import pandas as pd
+
+from src.rule_based_system.BadWordRule import BadWordRule
+
+
+def create_bad_word_rule(path, is_strict):
+    df = pd.read_csv(path, header=None)
+    df.columns = ['word']
+
+    bad_words = df['word'].values
+
+    return BadWordRule(bad_words, is_strict)

src/start_up/start_up_gibberish.py

@@ -0,0 +1,9 @@
+from src.gibberish_detection.GibberishDetector import GibberishDetector
+from gibberish_detector import detector
+from src.config import config
+
+
+def create_gibberish_detector():
+    model = detector.create_from_model(config['gibberish_model'])
+
+    return GibberishDetector(model)

src/start_up/start_up_rbs.py

@@ -0,0 +1,50 @@
+import re
+
+from urlextract import URLExtract
+
+from src.start_up.start_up_bad_words_rule import create_bad_word_rule
+from src.config import config
+from src.rule_based_system.HTMLRule import HTMLRule
+from src.rule_based_system.PersonalDetailsRule import PersonalDetailsRule
+from src.rule_based_system.RuleBasedSystem import RuleBasedSystem
+from src.rule_based_system.TextLengthRule import TextLengthRule
+from src.rule_based_system.UrlRule import UrlRule
+
+
+def create_strong_rbs() -> RuleBasedSystem:
+    text_length_rule = TextLengthRule()
+
+    url_rule = UrlRule(URLExtract())
+
+    mail_rule = PersonalDetailsRule([r'[\w.+-]+@[\w-]+\.[\w.-]+'], True)
+
+    strict_bad_word_rule = create_bad_word_rule(config['bad_words_strict'], True)
+
+    return RuleBasedSystem([
+        text_length_rule,  # todo: check if this make sense to add here, 500 was our own chosen max length
+        url_rule,
+        mail_rule,
+        strict_bad_word_rule
+    ])
+
+
+def create_weak_rbs() -> RuleBasedSystem:
+    phone_regex = r"(^\+[0-9]{2}|^\+[0-9]{2}\(0\)|^\(\+[0-9]{2}\)\(0\)|^00[0-9]{2}|^0)([0-9]{9}$|[0-9\-\s]{10}$)"
+    phone_home_local = re.compile(r".*?(\(?\d{3}\D{0,3}\d{2}\D{0,3}\d{2}).*?", re.S)
+    phone_home = re.compile(r".*?(\(?\d{3}\D{0,3}\d{3}\D{0,3}\d{2}\D{0,3}\d{2}).*?", re.S)
+    phone_mobile = re.compile(r".*?(\(?\d{2}\D{0,3}\d{3}\D{0,3}\d{3}\D{0,3}\d{2}).*?", re.S)
+    phone_mobile_international = re.compile(r".*?(\(?\d{3}\D{0,3}\d{3}\D{0,3}\d{3}\D{0,3}\d{2}).*?", re.S)
+
+    phone_regexes = [phone_regex, phone_home_local, phone_home, phone_mobile, phone_mobile_international]
+    phone_number_rule = PersonalDetailsRule(phone_regexes, False)
+
+    html_rule = HTMLRule()
+
+    ambiguous_bad_word_rule = create_bad_word_rule(config['bad_words_ambiguous'], False)
+
+    # rule systems
+    return RuleBasedSystem([
+        phone_number_rule,
+        html_rule,
+        ambiguous_bad_word_rule
+    ])