4 * - Assem Chelli, < assem [dot] ch [at] gmail >
5 * - Abdelkrim Aries <ab [underscore] aries [at] esi [dot] dz>
11 /* the Arabic letters in Unicode */
13 stringdef o '{U+0621}' // Hamza
14 stringdef ao '{U+0623}' // Hamza above Alef
15 stringdef ao_ '{U+0625}' // Hamza below Alef
16 stringdef a~ '{U+0622}' // Alef madda
17 stringdef wo '{U+0624}' // Hamza above waw
18 stringdef yo '{U+0626}' // Hamza above yeh
21 stringdef a '{U+0627}' // Alef
22 stringdef a_ '{U+0649}' // Alef Maksura
23 stringdef b '{U+0628}' // Beh
24 stringdef t_ '{U+0629}' // Teh_Marbuta
25 stringdef t '{U+062A}' // Teh
26 stringdef th '{U+062B}' // Theh
27 stringdef j '{U+062C}' // Jeem
28 stringdef h '{U+062D}' // Hah
29 stringdef x '{U+062E}' // Khah
30 stringdef d '{U+062F}' // Dal
31 stringdef dz '{U+0630}' // Thal
32 stringdef r '{U+0631}' // Reh
33 stringdef z '{U+0632}' // Zain
34 stringdef s '{U+0633}' // Seen
35 stringdef sh '{U+0634}' // Sheen
36 stringdef c '{U+0635}' // Sad
37 stringdef dh '{U+0636}' // Dad
38 stringdef tt '{U+0637}' // Tah
39 stringdef zh '{U+0638}' // Zah
40 stringdef i '{U+0639}' // Ain
41 stringdef gh '{U+063A}' // Ghain
42 stringdef f '{U+0641}' // Feh
43 stringdef q '{U+0642}' // Qaf
44 stringdef k '{U+0643}' // Kaf
45 stringdef l '{U+0644}' // Lam
46 stringdef m '{U+0645}' // Meem
47 stringdef n '{U+0646}' // Noon
48 stringdef e '{U+0647}' // Heh
49 stringdef w '{U+0648}' // Waw
50 stringdef y '{U+064A}' // Yeh
53 stringdef aan '{U+064B}' // FatHatan
54 stringdef uun '{U+064C}' // Dammatan
55 stringdef iin '{U+064D}' // Kasratan
56 stringdef aa '{U+064E}' // FatHa
57 stringdef uu '{U+064F}' // Damma
58 stringdef ii '{U+0650}' // Kasra
59 stringdef oo '{U+0652}' // Sukun
60 stringdef ~ '{U+0651}' // Shadda
62 // Hindu–Arabic numerals
63 stringdef 0 '{U+0660}'
64 stringdef 1 '{U+0661}'
65 stringdef 2 '{U+0662}'
66 stringdef 3 '{U+0663}'
67 stringdef 4 '{U+0664}'
68 stringdef 5 '{U+0665}'
69 stringdef 6 '{U+0666}'
70 stringdef 7 '{U+0667}'
71 stringdef 8 '{U+0668}'
72 stringdef 9 '{U+0669}'
76 stringdef _ '{U+0640}' // Kasheeda, Tatweel
79 stringdef o1 '{U+FE80}' // HAMZA
80 stringdef ao1 '{U+FE83}' // ALEF_HAMZA_ABOVE
81 stringdef ao2 '{U+FE84}' // ALEF_HAMZA_ABOVE
82 stringdef ao_1 '{U+FE87}' // ALEF_HAMZA_BELOW
83 stringdef ao_2 '{U+FE88}' // ALEF_HAMZA_BELOW
84 stringdef yo1 '{U+FE8B}' // YEH_HAMZA
85 stringdef yo2 '{U+FE8C}' // YEH_HAMZA
86 stringdef yo3 '{U+FE89}' // YEH_HAMZA
87 stringdef yo4 '{U+FE8A}' // YEH_HAMZA
88 stringdef a~1 '{U+FE81}' // ALEF_MADDA
89 stringdef a~2 '{U+FE82}' // ALEF_MADDA
90 stringdef wo1 '{U+FE85}' // WAW_HAMZA
91 stringdef wo2 '{U+FE86}' // WAW_HAMZA
92 stringdef a1 '{U+FE8D}' // ALEF
93 stringdef a2 '{U+FE8E}' // ALEF
94 stringdef b1 '{U+FE8F}' // BEH
95 stringdef b2 '{U+FE90}' // BEH
96 stringdef b3 '{U+FE91}' // BEH
97 stringdef b4 '{U+FE92}' // BEH
98 stringdef t_1 '{U+FE93}' // TEH_MARBUTA
99 stringdef t_2 '{U+FE94}' // TEH_MARBUTA
100 stringdef t1 '{U+FE97}' // TEH
101 stringdef t2 '{U+FE98}' // TEH
102 stringdef t3 '{U+FE95}' // TEH
103 stringdef t4 '{U+FE96}' // TEH
104 stringdef th1 '{U+FE9B}' // THEH
105 stringdef th2 '{U+FE9C}' // THEH
106 stringdef th3 '{U+FE9A}' // THEH
107 stringdef th4 '{U+FE99}' // THEH
108 stringdef j1 '{U+FE9F}' // JEEM
109 stringdef j2 '{U+FEA0}' // JEEM
110 stringdef j3 '{U+FE9D}' // JEEM
111 stringdef j4 '{U+FE9E}' // JEEM
112 stringdef h1 '{U+FEA3}' // HAH
113 stringdef h2 '{U+FEA4}' // HAH
114 stringdef h3 '{U+FEA1}' // HAH
115 stringdef h4 '{U+FEA2}' // HAH
116 stringdef x1 '{U+FEA7}' // KHAH
117 stringdef x2 '{U+FEA8}' // KHAH
118 stringdef x3 '{U+FEA5}' // KHAH
119 stringdef x4 '{U+FEA6}' // KHAH
120 stringdef d1 '{U+FEA9}' // DAL
121 stringdef d2 '{U+FEAA}' // DAL
122 stringdef dz1 '{U+FEAB}' // THAL
123 stringdef dz2 '{U+FEAC}' // THAL
124 stringdef r1 '{U+FEAD}' // REH
125 stringdef r2 '{U+FEAE}' // REH
126 stringdef z1 '{U+FEAF}' // ZAIN
127 stringdef z2 '{U+FEB0}' // ZAIN
128 stringdef s1 '{U+FEB3}' // SEEN
129 stringdef s2 '{U+FEB4}' // SEEN
130 stringdef s3 '{U+FEB1}' // SEEN
131 stringdef s4 '{U+FEB2}' // SEEN
132 stringdef sh1 '{U+FEB7}' // SHEEN
133 stringdef sh2 '{U+FEB8}' // SHEEN
134 stringdef sh3 '{U+FEB5}' // SHEEN
135 stringdef sh4 '{U+FEB6}' // SHEEN
136 stringdef c1 '{U+FEBB}' // SAD
137 stringdef c2 '{U+FEBC}' // SAD
138 stringdef c3 '{U+FEB9}' // SAD
139 stringdef c4 '{U+FEBA}' // SAD
140 stringdef dh1 '{U+FEBF}' // DAD
141 stringdef dh2 '{U+FEC0}' // DAD
142 stringdef dh3 '{U+FEBD}' // DAD
143 stringdef dh4 '{U+FEBE}' // DAD
144 stringdef tt1 '{U+FEC3}' // TAH
145 stringdef tt2 '{U+FEC4}' // TAH
146 stringdef tt3 '{U+FEC1}' // TAH
147 stringdef tt4 '{U+FEC2}' // TAH
148 stringdef zh1 '{U+FEC7}' // ZAH
149 stringdef zh2 '{U+FEC8}' // ZAH
150 stringdef zh3 '{U+FEC5}' // ZAH
151 stringdef zh4 '{U+FEC6}' // ZAH
152 stringdef i1 '{U+FECB}' // AIN
153 stringdef i2 '{U+FECC}' // AIN
154 stringdef i3 '{U+FEC9}' // AIN
155 stringdef i4 '{U+FECA}' // AIN
156 stringdef gh1 '{U+FECF}' // GHAIN
157 stringdef gh2 '{U+FED0}' // GHAIN
158 stringdef gh3 '{U+FECD}' // GHAIN
159 stringdef gh4 '{U+FECE}' // GHAIN
160 stringdef f1 '{U+FED3}' // FEH
161 stringdef f2 '{U+FED4}' // FEH
162 stringdef f3 '{U+FED1}' // FEH
163 stringdef f4 '{U+FED2}' // FEH
164 stringdef q1 '{U+FED7}' // QAF
165 stringdef q2 '{U+FED8}' // QAF
166 stringdef q3 '{U+FED5}' // QAF
167 stringdef q4 '{U+FED6}' // QAF
168 stringdef k1 '{U+FEDB}' // KAF
169 stringdef k2 '{U+FEDC}' // KAF
170 stringdef k3 '{U+FED9}' // KAF
171 stringdef k4 '{U+FEDA}' // KAF
172 stringdef l1 '{U+FEDF}' // LAM
173 stringdef l2 '{U+FEE0}' // LAM
174 stringdef l3 '{U+FEDD}' // LAM
175 stringdef l4 '{U+FEDE}' // LAM
176 stringdef m1 '{U+FEE3}' // MEEM
177 stringdef m2 '{U+FEE4}' // MEEM
178 stringdef m3 '{U+FEE1}' // MEEM
179 stringdef m4 '{U+FEE2}' // MEEM
180 stringdef n1 '{U+FEE7}' // NOON
181 stringdef n2 '{U+FEE8}' // NOON
182 stringdef n3 '{U+FEE5}' // NOON
183 stringdef n4 '{U+FEE6}' // NOON
184 stringdef e1 '{U+FEEB}' // HEH
185 stringdef e2 '{U+FEEC}' // HEH
186 stringdef e3 '{U+FEE9}' // HEH
187 stringdef e4 '{U+FEEA}' // HEH
188 stringdef w1 '{U+FEED}' // WAW
189 stringdef w2 '{U+FEEE}' // WAW
190 stringdef a_1 '{U+FEEF}' // ALEF_MAKSURA
191 stringdef a_2 '{U+FEF0}' // ALEF_MAKSURA
192 stringdef y1 '{U+FEF3}' // YEH
193 stringdef y2 '{U+FEF4}' // YEH
194 stringdef y3 '{U+FEF1}' // YEH
195 stringdef y4 '{U+FEF2}' // YEH
197 // Ligatures Lam-Alef
198 stringdef la '{U+FEFB}' // LAM_ALEF
199 stringdef la2 '{U+FEFC}' // LAM_ALEF
200 stringdef lao '{U+FEF7}' // LAM_ALEF_HAMZA_ABOVE
201 stringdef lao2 '{U+FEF8}' // LAM_ALEF_HAMZA_ABOVE
202 stringdef lao_ '{U+FEF9}' // LAM_ALEF_HAMZA_BELOW
203 stringdef lao_2 '{U+FEFA}' // LAM_ALEF_HAMZA_BELOW
204 stringdef la~ '{U+FEF5}' // LAM_ALEF_MADDA_ABOVE
205 stringdef la~2 '{U+FEF6}' // LAM_ALEF_MADDA_ABOVE
222 Suffix_All_alef_maqsura
247 define Normalize_pre as (
251 '{aan}' '{uun}' '{iin}' '{aa}' '{uu}' '{ii}' '{oo}' '{~}'( delete ) // strip vocalization
252 '{_}' ( delete ) // strip kasheeda
254 // Hindu–Arabic numerals
267 '{o1}' ( <- '{o}' ) // HAMZA
268 '{ao1}' '{ao2}' ( <- '{ao}' ) // ALEF_HAMZA_ABOVE
269 '{ao_1}' '{ao_2}' ( <- '{ao_}' ) // ALEF_HAMZA_BELOW
270 '{yo1}' '{yo2}' '{yo3}' '{yo4}' ( <- '{yo}' ) // YEH_HAMZA
271 '{a~1}' '{a~2}'( <- '{a~}' ) // ALEF_MADDA
272 '{wo1}' '{wo2}'( <- '{wo}' ) // WAW_HAMZA
273 '{a1}' '{a2}' ( <- '{a}' ) // ALEF
274 '{b1}' '{b2}' '{b3}' '{b4}' ( <- '{b}' ) // BEH
275 '{t_1}' '{t_2}' ( <- '{t_}' ) // TEH_MARBUTA
276 '{t1}' '{t2}' '{t3}' '{t4}' ( <- '{t}' ) // TEH
277 '{th1}' '{th2}' '{th3}' '{th4}' ( <- '{th}' ) // THEH
278 '{j1}' '{j2}' '{j3}' '{j4}'( <- '{j}' ) // JEEM
279 '{h1}' '{h2}' '{h3}' '{h4}' ( <- '{h}' ) // HAH
280 '{x1}' '{x2}' '{x3}' '{x4}'( <- '{x}' ) // KHAH
281 '{d1}' '{d2}' ( <- '{d}' ) // DAL
282 '{dz1}''{dz2}' ( <- '{dz}' ) // THAL
283 '{r1}' '{r2}'( <- '{r}' ) // REH
284 '{z1}' '{z2}' ( <- '{z}' ) // ZAIN
285 '{s1}' '{s2}' '{s3}' '{s4}'( <- '{s}' ) // SEEN
286 '{sh1}' '{sh2}' '{sh3}' '{sh4}' ( <- '{sh}' ) // SHEEN
287 '{c1}' '{c2}' '{c3}' '{c4}'( <- '{c}' ) // SAD
288 '{dh1}' '{dh2}' '{dh3}' '{dh4}'( <- '{dh}' ) // DAD
289 '{tt1}' '{tt2}' '{tt3}' '{tt4}' ( <- '{tt}' ) // TAH
290 '{zh1}' '{zh2}' '{zh3}' '{zh4}'( <- '{zh}' ) // ZAH
291 '{i1}' '{i2}' '{i3}' '{i4}'( <- '{i}' ) // AIN
292 '{gh1}' '{gh2}' '{gh3}' '{gh4}'( <- '{gh}' ) // GHAIN
293 '{f1}' '{f2}' '{f3}' '{f4}' ( <- '{f}' ) // FEH
294 '{q1}' '{q2}' '{q3}' '{q4}' ( <- '{q}' ) // QAF
295 '{k1}' '{k2}' '{k3}' '{k4}'( <- '{k}' ) // KAF
296 '{l1}' '{l2}' '{l3}' '{l4}'( <- '{l}' ) // LAM
297 '{m1}' '{m2}' '{m3}' '{m4}' ( <- '{m}' ) // MEEM
298 '{n1}' '{n2}' '{n3}' '{n4}'( <- '{n}' ) // NOON
299 '{e1}' '{e2}' '{e3}' '{e4}' ( <- '{e}' ) // HEH
300 '{w1}' '{w2}' ( <- '{w}' ) // WAW
301 '{a_1}' '{a_2}' ( <- '{a_}' ) // ALEF_MAKSURA
302 '{y1}' '{y2}' '{y3}' '{y4}' ( <- '{y}' ) // YEH
304 // Ligatures Lam-Alef
305 '{la}' '{la2}' (<- '{l}{a}')
306 '{lao}' '{lao2}' (<- '{l}{ao}')
307 '{lao_}' '{lao_2}' (<- '{l}{ao_}')
308 '{la~}' '{la~2}' (<- '{l}{a~}')
317 define Normalize_post as (
320 // normalize last hamza
323 '{ao}''{ao_}' '{a~}' ( <- '{o}')
332 // normalize other hamza's
334 '{ao}''{ao_}' '{a~}' ( <- '{a}')
347 '{b}{a}{l}' '{k}{a}{l}' ($(len > 4) set is_noun unset is_verb set is_defined)
348 '{l}{l}' '{a}{l}' ($(len > 3) set is_noun unset is_verb set is_defined)
354 define Prefix_Step1 as (
356 '{ao}{ao}' ($(len > 3) <- '{ao}' )
357 '{ao}{a~}' ($(len > 3) <- '{a~}' )
358 '{ao}{wo}' ($(len > 3) <- '{ao}' )
359 '{ao}{a}' ($(len > 3) <- '{a}' )
360 '{ao}{ao_}' ($(len > 3) <- '{ao_}' )
361 // '{ao}' ($(len > 3) delete) //rare case
365 define Prefix_Step2 as (
369 '{f}' ($(len > 3) delete)
370 '{w}' ($(len > 3) delete)
374 define Prefix_Step3a_Noun as ( // it is noun and defined
376 '{b}{a}{l}' '{k}{a}{l}' ($(len > 5) delete)
377 '{l}{l}' '{a}{l}' ($(len > 4) delete)
381 define Prefix_Step3b_Noun as ( // probably noun and defined
382 not '{b}{a}' // exception
384 '{b}' ($(len > 3) delete)
385 // '{k}' '{l}' ($(len > 3) delete) // BUG: cause confusion
386 '{b}{b}' ($(len > 3) <- '{b}' )
387 '{k}{k}' ($(len > 3) <- '{k}' )
392 define Prefix_Step3_Verb as (
394 //'{s}' ($(len > 4) delete)// BUG: cause confusion
395 '{s}{y}' ($(len > 4) <- '{y}' )
396 '{s}{t}' ($(len > 4) <- '{t}')
397 '{s}{n}' ($(len > 4) <- '{n}')
398 '{s}{ao}' ($(len > 4) <- '{ao}')
402 define Prefix_Step4_Verb as (
404 '{y}{s}{t}' '{n}{s}{t}' '{t}{s}{t}' ($(len > 4) set is_verb unset is_noun <- '{a}{s}{t}' )
411 define Suffix_Noun_Step1a as (
413 '{y}' '{k}' '{e}' ($(len >= 4) delete)
414 '{n}{a}' '{k}{m}' '{e}{a}' '{e}{n}' '{e}{m}' ($(len >= 5) delete)
415 '{k}{m}{a}' '{e}{m}{a}' ($(len >= 6) delete)
418 define Suffix_Noun_Step1b as (
420 '{n}' ($(len > 5) delete)
424 define Suffix_Noun_Step2a as (
426 '{a}' '{y}' '{w}' ($(len > 4) delete)
430 define Suffix_Noun_Step2b as (
432 '{a}{t}' ($(len >= 5) delete)
436 define Suffix_Noun_Step2c1 as (
438 '{t}' ($(len >= 4) delete)
441 define Suffix_Noun_Step2c2 as ( // feminine t_
443 '{t_}' ($(len >= 4) delete)
446 define Suffix_Noun_Step3 as ( // ya' nisbiya
448 '{y}' ($(len >= 3) delete)
452 define Suffix_Verb_Step1 as (
454 '{e}' '{k}' ($(len >= 4) delete)
455 '{n}{y}' '{n}{a}' '{e}{a}' '{e}{m}' '{e}{n}' '{k}{m}' '{k}{n}' ($(len >= 5) delete)
456 '{e}{m}{a}' '{k}{m}{a}' '{k}{m}{w}'($(len >= 6) delete)
459 define Suffix_Verb_Step2a as (
461 '{t}' ($(len >= 4) delete)
462 '{a}' '{n}' '{y}' ($(len >= 4) delete)
463 '{n}{a}' '{t}{a}' '{t}{n}' ($(len >= 5) delete)// past
464 '{a}{n}' '{w}{n}' '{y}{n}' ($(len > 5) delete) // present
465 '{t}{m}{a}' ($(len >= 6) delete)
469 define Suffix_Verb_Step2b as (
471 '{w}{a}' '{t}{m}' ($(len >= 5) delete)
476 define Suffix_Verb_Step2c as (
478 '{w}' ($(len >= 4) delete)
479 '{t}{m}{w}' ($(len >= 6) delete)
483 define Suffix_All_alef_maqsura as (
485 '{a_}' ( <- '{y}' ) // spell error
486 // '{a_}' ( delete ) // if noun > 3
487 // '{a_}' ( <- '{a}') // if verb
493 // set initial values
498 // guess type and properties
501 // normalization pre-stemming
513 (atleast 1 Suffix_Verb_Step1)
514 ( Suffix_Verb_Step2a or Suffix_Verb_Step2c or next)
516 or Suffix_Verb_Step2b
517 or Suffix_Verb_Step2a
527 or (not is_defined Suffix_Noun_Step1a (
529 or Suffix_Noun_Step2b
530 or Suffix_Noun_Step2c1
532 or (Suffix_Noun_Step1b (
534 or Suffix_Noun_Step2b
535 or Suffix_Noun_Step2c1))
536 or (not is_defined Suffix_Noun_Step2a)
537 or (Suffix_Noun_Step2b)
544 // Suffixes for alef maqsura
545 or Suffix_All_alef_maqsura
554 or (is_noun Prefix_Step3b_Noun)
555 or (is_verb try Prefix_Step3_Verb Prefix_Step4_Verb)
559 // normalization post-stemming