?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
CMiniLexicon::FindMajorSignatures(): use log file routines
[linguistica.git] / Template.cpp
blob32784cdad9dcd2c2c8907eabf32965d1db979edd
1 // Implementation of CTemplate methods
2 // Copyright © 2009 The University of Chicago
3 #include "Template.h"
4
5 #include <QMessageBox>
6 #include <Q3TextStream>
7 #include <QIODevice>
8 #include <QFile>
9 #include "Lexicon.h"
10 #include "Alignment.h"
11 #include "TemplateCollection.h"
12 #include "WordCollection.h"
13 #include "generaldefinitions.h"
14 #include "Parse.h"
15 #include "log2.h"
16
17 extern double g_Lambda ;
18
19 CTemplate::CTemplate(int NumberOfColumns)
20 {
21 m_NumberOfColumns = NumberOfColumns;
22 m_Columns = new CParse*[ m_NumberOfColumns ];
23
24 for (int i = 0; i < NumberOfColumns; i++)
25 {
26 m_Columns[i] = new CParse();
27 }
28 m_Complexity = 0;
29 m_WordsTotalComplexity = 0;
30 m_TemplateNumber = 0;
31
32 m_IsDeleted = false;
33 m_ModifiedColumn = -1;
34 m_IsNewAfterCollapse1 = false;
35 m_StemColumnInCollapse1 =-1;
36 m_SwitchOfSortingValue = false;
37
38
39 }
40 CTemplate::CTemplate(CTemplate& Template) : CStem ( Template.Display() )
41 {
42
43 m_NumberOfColumns = Template.m_NumberOfColumns;
44 m_Columns = new CParse* [ m_NumberOfColumns ];
45 for (int i = 0; i < m_NumberOfColumns; i++)
46 {
47
48 m_Columns[i] = new CParse ( *Template.m_Columns[i]);
49 Q_ASSERT ( m_Columns[i]->Size() > 0 );
50 }
51
52 m_WordsTotalComplexity = Template.GetWordsTotalComplexity();
53 m_Complexity = Template.GetComplexity();
54 m_TemplateNumber = Template.m_TemplateNumber;
55
56 m_IsDeleted = Template.m_IsDeleted;
57 m_ModifiedColumn = Template.m_ModifiedColumn;
58 m_IsNewAfterCollapse1 = Template.m_IsNewAfterCollapse1;
59 m_StemColumnInCollapse1 = Template.m_StemColumnInCollapse1;
60 m_SwitchOfSortingValue = Template.m_SwitchOfSortingValue;
61
62 }
63
64 CTemplate::CTemplate(CAlignment* pAlign) : CStem ( pAlign->SpellOut() )
65 {
66
67 int loc1=0, loc2=0;
68 int StartLoc1, StartLoc2;
69 int col = 0;
70
71 if ( pAlign->m_Slips != 1 ) { return; }
72
73 QString debugstring1, debugstring2;
74 const char* CCDebugString1, *CCDebugString2;
75
76 debugstring1 = pAlign ->m_Str1 ->GetKey().Display();
77 CCDebugString1 = debugstring1.ascii();
78 debugstring2 = pAlign ->m_Str2 ->GetKey().Display();
79 CCDebugString2 = debugstring2.ascii();
80
81
82 m_NumberOfColumns = pAlign->m_Spans;
83 m_Columns = new CParse*[ m_NumberOfColumns ];
84
85 for (int c = 0; c < m_NumberOfColumns; c++)
86 {
87 m_Columns[c] = new CParse();
88 }
89
90
91 // it always does this loop at least once, because of initial "#" which is shared
92 while ( pAlign->PerfectMatch (loc1, loc2) )
93 {
94 loc1++;
95 loc2++;
96 }
97
98 if ( loc1 > 1 )
99 { // there is an initial shared span...
100 m_Columns[col]->Append( CStringSurrogate (pAlign->m_Str1->GetKeyPointer(),0 , loc1 ) );
101 col++;
102 StartLoc1 = loc1;
103 StartLoc2 = loc2;
104 }
105 else // there is no initial shared span, and we'll back up to put the # in each piece
106 {
107 StartLoc1 = loc1 - 1;
108 StartLoc2 = loc2 - 1;
109 }
110
111
112
113 while ( loc1 < pAlign->m_Length1 &&
114 pAlign->m_Str2->GetChar( pAlign->m_Match1[loc1] ) != pAlign->m_Str1->GetChar( loc1 )
115 )
116 {
117 loc1++;
118 }
119
120 while ( loc2 < pAlign->m_Length2 &&
121 pAlign->m_Str1->GetChar( pAlign->m_Match2[loc2] ) != pAlign->m_Str2->GetChar(loc2) )
122 {
123 loc2++;
124 }
125
126
127 CStringSurrogate Piece1 ( pAlign->m_Str1->GetKeyPointer(), StartLoc1 , loc1 - StartLoc1 );
128 CStringSurrogate Piece2 ( pAlign->m_Str2->GetKeyPointer(), StartLoc2 , loc2 - StartLoc2 );
129
130
131 if ( Piece1.GetLength () == 0 )
132 {
133 // m_Columns[col]->AppendInAlphabeticalOrder( CStringSurrogate(QString("NULL").unicode(), 0, 4));
134 m_Columns[col]->Append(CStringSurrogate(QString("NULL").unicode(), 0, 4));
135 }
136 else
137 {
138 m_Columns[col]->Append(Piece1);
139 // m_Columns[col]->AppendInAlphabeticalOrder( Piece1, true);
140 }
141
142
143 if ( Piece2.GetLength () == 0 )
144 {
145 // m_Columns[col]->AppendInAlphabeticalOrder( CStringSurrogate(QString("NULL").unicode(), 0, 4), true );
146 m_Columns[col]->Append(
147 CStringSurrogate(QString("NULL").unicode(), 0, 4));
148 }
149 else
150 {
151 // m_Columns[col]->AppendInAlphabeticalOrder( Piece2, true);
152 m_Columns[col]->Append(Piece2);
153 }
154
155 // yuhuask when slip == 1 Only have three cases: yn or yny
156 if ( col + 1 < m_NumberOfColumns )
157 {
158 col++;
159 // m_Columns[col]->AppendInAlphabeticalOrder( CStringSurrogate (pAlign->m_Str1->GetKeyPointer(), loc1,pAlign->m_Str1->GetKeyLength() - loc1 ), true );
160 m_Columns[col]->Append(CStringSurrogate (pAlign->m_Str1->GetKeyPointer(), loc1,pAlign->m_Str1->GetKeyLength() - loc1));
161 }
162
163 for (int i = 1; i < m_NumberOfColumns; i++)
164 {
165 Q_ASSERT(m_Columns[i]->Size() > 0 );
166 }
167
168
169 Q_ASSERT (m_Columns[0]->GetChar(0) == '#');
170 m_TemplateNumber = 0;
171
172
173 m_ModifiedColumn = -1;
174 m_IsDeleted = false;
175 m_IsNewAfterCollapse1 = false;
176 m_StemColumnInCollapse1 =-1;
177 m_SwitchOfSortingValue = false;
178 return ;
179
180 }
181
182
183 void CTemplate::operator= (CAlignment* pAlign)
184 {
185
186 int loc1= 0, loc2=0;
187 int StartLoc1, StartLoc2;
188 int col = 0;
189
190 if ( m_NumberOfColumns )
191 {
192 for (int i = 0; i < m_NumberOfColumns; i++)
193 {
194 if ( m_Columns[i] ) delete m_Columns[i];
195 }
196 }
197
198 if ( pAlign->m_Slips != 1 ) { return; }
199
200 m_NumberOfColumns = pAlign->m_Spans;
201
202 m_Columns = new CParse*[ m_NumberOfColumns ];
203
204 for (int c = 0; c < m_NumberOfColumns; c++)
205 {
206 m_Columns[c] = new CParse();
207 }
208
209
210 while ( pAlign->m_Str2->GetChar( loc2 ) == pAlign->m_Str1->GetChar( loc1 ) )
211 {
212 loc1++;
213 loc2++;
214 }
215
216 if ( loc1 > 1 ) // they agree at the beginning, up to loc1 - 1;
217 {
218 m_Columns[col]->Append( CStringSurrogate (pAlign->m_Str1->GetKeyPointer(), 0, loc1) ); // yuhuask should be loc1 not loc1 -1
219 }
220
221
222
223
224
225 col++;
226 StartLoc1 = loc1;
227 StartLoc2 = loc2;
228
229 while ( loc1 < pAlign->m_Length1 &&
230 pAlign->m_Str2->GetChar( pAlign->m_Match1[loc1] ) != pAlign->m_Str1->GetChar( loc1 )
231 )
232 {
233 loc1++;
234 }
235
236 while ( loc2 < pAlign->m_Length2 &&
237 pAlign->m_Str1->GetChar( pAlign->m_Match2[loc2] ) != pAlign->m_Str2->GetChar( loc2 ) )
238 {
239 loc2++;
240 }
241
242
243 CStringSurrogate Piece1 ( pAlign->m_Str1->GetKeyPointer(), StartLoc1, loc1 - StartLoc1 ); // yuhuask same problem StartLoc1 - 1 ?
244 CStringSurrogate Piece2 ( pAlign->m_Str2->GetKeyPointer(), StartLoc2, loc2 - StartLoc2 );
245
246
247
248 if ( Piece1.GetLength () == 0 )
249 {
250 m_Columns[col]->Append( CStringSurrogate(QString("NULL").unicode(), 0, 4) );
251 }
252 else
253 {
254 m_Columns[col]->Append( Piece1);
255 }
256
257
258 if ( Piece2.GetLength () == 0 )
259 {
260 m_Columns[col]->Append( CStringSurrogate(QString("NULL").unicode(), 0, 4) );
261 }
262 else
263 {
264 m_Columns[col]->Append( Piece2);
265 }
266
267 col++;
268
269 m_Columns[col]->Append( CStringSurrogate (pAlign->m_Str1->GetKeyPointer(), loc1, pAlign->m_Str1->GetKeyLength() - loc1 ) );
270
271 m_TemplateNumber = 0;
272
273 return ;
274
275 }
276
277
278
279 CTemplate::~CTemplate(void)
280 {
281
282 for (int i = 0; i < m_NumberOfColumns; i++)
283 {
284 delete m_Columns[i];
285 }
286 delete m_Columns;
287
288 }
289
290 //------------------------------------------------------
291
292 CParse* CTemplate::GetColumn(int n )
293 {
294
295 Q_ASSERT ( n >= 0 );
296
297
298 if ( n < m_NumberOfColumns && n >= 0 )
299 {
300 return m_Columns[n];
301 }
302 return NULL;
303
304 }
305
306 //------------------------------------------------------
307
308
309
310
311
312
313 QString CTemplate::Display()
314 {
315 QString Return;
316
317 for (int i = 0; i < m_NumberOfColumns; i++)
318 {
319 Return += m_Columns[i]->Display() + "_";
320 }
321
322 return Return;
323 }
324
325 /* From JG's original:
326
327 void CTemplate::Display(CListCtrl& List, int& LineNumber)
328 {
329 int NumberOfRows = 0;
330 for (int col = 0; col < m_NumberOfColumns; col++)
331 {
332 if (m_Columns[col]->Size() > NumberOfRows ) { NumberOfRows = m_Columns[col]->Size(); }
333 }
334
335 for (int row = 0; row < NumberOfRows+1; row++)
336 {
337 List.InsertItem (LineNumber + row, CString ("") );
338 }
339
340 List.SetItemText ( LineNumber, 0, IntToString( (int) GetSortingQuantity() ) );
341
342
343
344
345 for ( row = 0; row < NumberOfRows; row++)
346 {
347 for (int col = 0; col < m_NumberOfColumns; col++)
348 {
349 if ( m_Columns[col]->Size() > row )
350 {
351 List.SetItemText(LineNumber+row, col + 1, m_Columns[col]->GetAtCString(row+1) );
352 }
353 }
354 }
355
356 LineNumber += row;
357
358
359 }
360
361 */
362 /*
363 QString CTemplate::DisplayFirstLine()
364 {
365 QString Return;
366
367 for (int i = 0; i < m_NumberOfColumns; i++)
368 {
369 Return += m_Columns[i]->GetPiece(1).Display() + "_";
370 }
371
372 return Return;
373 }
374 */
375 /*
376 void CTemplate::Display(CListCtrl& List, int& LineNumber)
377 {
378
379 int NumberOfRows = 0;
380 for (int col = 0; col < m_NumberOfColumns; col++)
381 {
382 if (m_Columns[col]->Size() > NumberOfRows ) { NumberOfRows = m_Columns[col]->Size(); }
383 }
384
385 for (int row = 0; row < NumberOfRows+1; row++)
386 {
387 List.InsertItem (LineNumber + row, CString ("") );
388 }
389
390 // if ( GetSortingQuantity() > 15000 || GetSortingQuantity() < 0 )
391 // {
392 // ofstream out ("c:\\4_tests\\IntToString.txt", ios::app );
393 // out << endl << endl; OutputForFile (out);
394 // out << endl << GetSortingQuantity();
395 // out << " " << IntToString ( (int) GetSortingQuantity () );
396 // }
397 List.SetItemText ( LineNumber, 0, IntToString( (int) GetSortingQuantity() ) );
398
399 for ( row = 0; row < NumberOfRows; row++)
400 {
401 for (int col = 0; col < m_NumberOfColumns; col++)
402 {
403 if ( m_Columns[col]->Size() > row )
404 {
405 List.SetItemText(LineNumber+row, col + 1, m_Columns[col]->GetAtCString(row+1) );
406 }
407 }
408 }
409
410 LineNumber += row;
411
412
413 }
414 */
415
416 void CTemplate::ListDisplay(Q3ListView* List)
417 {
418 GetSortingQuantity();
419 static_cast<void>(new CTemplateListViewItem(List, this));
420 }
421
422 int CTemplate::GetVerticalColumn()
423 {
424
425 for (int i = 0; i < m_NumberOfColumns; i++)
426 {
427 if ( m_Columns[i]->Size() > 1 ) return i;
428 }
429
430 return -1;
431 }
432 void CTemplate::AddToColumn(CParse& Morphemes, int n)
433 {
434
435 for (int i = 1; i <= Morphemes.Size(); i++)
436 {
437 // m_Columns[n]->AppendInAlphabeticalOrder ( Morphemes.GetPiece(i), true );
438 m_Columns[n]->Append(Morphemes.GetPiece(i));
439 }
440
441 }
442 void CTemplate::AddToColumn(CStringSurrogate& SS, int n)
443 {
444 // m_Columns[n]->AppendInAlphabeticalOrder ( SS, true );
445 m_Columns[n]->Append(SS);
446
447 }
448
449
450
451
452 float CTemplate::GetSortingQuantity() const
453 {
454
455
456 { ComputeComplexity(); }
457
458 { ComputeWordsTotalComplexity(); }
459
460
461 Q_ASSERT ( m_WordsTotalComplexity - m_Complexity < 55000);
462
463
464 m_CurrentSortComplexity = m_WordsTotalComplexity - m_Complexity;
465
466 // I set up a switch flag to indicate that we switch the sorting value
467 if ( m_SwitchOfSortingValue )
468 {
469 int NumberOfStems =0;
470 int i;
471 CParse* OneColumn;
472
473
474
475 // Roughly get the number of stems
476 for ( i = 0; i < m_NumberOfColumns; i++)
477 {
478 OneColumn = m_Columns[i];
479 if ( OneColumn ->Size() > NumberOfStems)
480 {
481 NumberOfStems = OneColumn ->Size();
482 }
483 }
484
485
486 m_NewSortComplexity = (m_WordsTotalComplexity - m_Complexity)/(float)NumberOfStems;
487 }
488 else
489 {
490 m_NewSortComplexity = m_WordsTotalComplexity - m_Complexity;
491 }
492
493
494
495 return float ( m_NewSortComplexity);
496 //return float ( m_WordsTotalComplexity - m_Complexity );
497
498
499
500 }
501
502
503 /*
504 float CTemplate::GetSortingQuantity()
505 {
506
507 // if ( m_Complexity <= 0 )
508 { ComputeComplexity(); }
509
510 // if ( m_WordsTotalComplexity <= 0 )
511 { ComputeWordsTotalComplexity(); }
512
513 // ofstream out ("c:\\4_tests\\TemplateDisplay.txt", ios::app );
514
515 Q_ASSERT ( m_WordsTotalComplexity - m_Complexity < 55000);
516
517
518 // if ( m_WordsTotalComplexity < 0 || m_WordsTotalComplexity > 15000)
519 // {
520 // OutputForFile(out);
521 // }
522 // if ( m_Complexity < 0 || m_Complexity > 15000)
523 // {
524 // OutputForFile(out);
525 // }
526
527 m_SortingComplexity = m_WordsTotalComplexity - m_Complexity;
528
529 return float ( m_WordsTotalComplexity - m_Complexity );
530
531
532 }
533 */
534
535 void CTemplate::AddAlignment(CAlignment* pAlignment)
536 {
537 struct not_implemented { };
538 throw not_implemented();
539 static_cast<void>(pAlignment);
540
541 // m_Alignments.AddTail( pAlignment);
542 // yuhuask ?
543 }
544
545 bool CTemplate::ShouldConflate ( CTemplate* pAlignment, int& ConflateColumn )
546 {
547
548 /* Works only with templates of 2 or 3 columns.
549 If two alignments share a column, they are collapsed. */
550
551
552 if ( m_NumberOfColumns != pAlignment->m_NumberOfColumns ) { return FALSE; }
553 if ( m_NumberOfColumns == 2 )
554 {
555 if (// m_Columns[0]->Size() > 1 &&
556 *m_Columns[0] == pAlignment->m_Columns[0] )
557 {
558 ConflateColumn = 1;
559 return TRUE;
560 }
561 else if (// m_Columns[1]->Size() > 1 &&
562 *m_Columns[1] == pAlignment->m_Columns[1] )
563 {
564 ConflateColumn = 0;
565 return TRUE;
566 }
567 else
568 {
569 return FALSE;
570 }
571 } // end of Size = 2;
572 else if ( m_NumberOfColumns == 3 )
573 {
574 if ( //m_Columns[0]->Size() > 1 &&
575 *m_Columns[0] == pAlignment->m_Columns[0] &&
576 //m_Columns[2]->Size() > 1 &&
577 *m_Columns[2] == pAlignment->m_Columns[2]
578 )
579 {
580 ConflateColumn = 1;
581 return TRUE;
582 }
583 else if (// m_Columns[1]->Size() > 1 &&
584 *m_Columns[1] == pAlignment->m_Columns[1] &&
585 // m_Columns[2]->Size() > 1 &&
586 *m_Columns[2] == pAlignment->m_Columns[2] )
587 {
588 ConflateColumn = 0;
589 return TRUE;
590 }
591 else if (// m_Columns[0]->Size() > 1 &&
592 *m_Columns[0] == pAlignment->m_Columns[0] &&
593 // m_Columns[1]->Size() > 1 &&
594 *m_Columns[1] == pAlignment->m_Columns[1] )
595 {
596 ConflateColumn = 2;
597 return TRUE;
598 }
599 else
600 {
601 return FALSE;
602 }
603
604 } // end of Size = 3;
605
606 return FALSE;
607
608
609 }
610
611 void CTemplate::ConflateWith ( CTemplate* pOther, int Column )
612 {
613 CParse& their_column = *pOther->m_Columns[Column];
614 CParse& my_column = *m_Columns[Column];
615
616 for (int i = 1; i <= their_column.Size(); ++i)
617 my_column.Append(their_column.GetPiece(i));
618 }
619
620 bool CTemplate::Readjust(CTemplateCollection* TempTemplates, int MinCount )
621 {
622 // bool AllSame = TRUE;
623 bool Return = FALSE;
624 CWordCollection Words;
625
626
627
628 ComputeWordsTotalComplexity();
629
630 int i;
631 for (i = 1; i < m_NumberOfColumns; i++)
632 {
633 if ( (int) m_Columns[i]->Size() < MinCount )
634 {
635 continue;
636 }
637
638 Words.RemoveAll();
639 for (int w = 1; w <= (int) m_Columns[i]->Size(); w++)
640 {
641 Words << m_Columns[i]->GetPiece(w).Display();
642 }
643 CStringSurrogate ssPrefix = Words.FindMaximalMajorityPrefix();
644 if ( ssPrefix.GetLength() == 0 ) { continue; }
645
646 //------------------------------------------
647 // if we have some material that might potentially be
648 // moved leftward, let's calculate the complexity in that
649 // new situation.
650 //
651 //------------------------------------------
652
653
654 CParse RightColumn = *m_Columns[ i ];
655 CParse RightColumnOtherMorphemes = RightColumn;
656 CParse LeftColumn = *m_Columns[i-1];
657
658 RightColumn .RemovePiecesThatDoNotBegin ( ssPrefix );
659 RightColumn .RemovePrefixFromAllPieces ( ssPrefix );
660 RightColumnOtherMorphemes
661 .RemovePiecesThatBegin( ssPrefix );
662
663 LeftColumn .SuffixToAllPieces2 ( ssPrefix );
664
665 double OldComplexity = m_Columns[i]->ComputeComplexity() +
666 m_Columns[i-1]->ComputeComplexity();
667 double NewComplexity = RightColumn.ComputeComplexity() +
668 LeftColumn.ComputeComplexity();
669
670 if (RightColumnOtherMorphemes.Size() > 0 )
671 {
672 NewComplexity += RightColumnOtherMorphemes.ComputeComplexity() +
673 m_Columns[i-1]->ComputeComplexity();
674 }
675
676
677
678 if ( NewComplexity < OldComplexity )
679 {
680 if ( RightColumnOtherMorphemes.Size() > 0 )
681 {
682 CTemplate TempTemplate ( *this );
683 *TempTemplate.GetColumn( i-1 ) = *m_Columns[ i-1 ];
684 *TempTemplate.GetColumn( i ) = RightColumnOtherMorphemes;
685
686 TempTemplates->AddTemplate ( &TempTemplate);
687 }
688
689 *m_Columns[i] = RightColumn;
690 *m_Columns[i-1] = LeftColumn;
691 }
692 }
693
694
695
696 CStringSurrogate TempSS;
697
698
699 for ( i = 1; i < m_NumberOfColumns ; i++)
700 {
701
702 if ( (int) m_Columns[i]->Size() < MinCount )
703 {
704 continue;
705 }
706
707 Words.RemoveAll();
708 for (int w = 1; w <= (int) m_Columns[i-1]->Size(); w++)
709 {
710 TempSS = m_Columns[i-1]->GetPiece(w);
711 TempSS.SetBackwards();
712 Words << TempSS;
713 }
714 TempSS = Words.FindMaximalMajorityPrefix();
715 TempSS.SetBackwards();
716 CStringSurrogate ssSuffix ( TempSS );
717
718 if ( ssSuffix.GetLength() == 0 ) { continue; }
719
720 //------------------------------------------
721 //
722 // if we have some material that might potentially be
723 // moved rightward, let's calculate the complexity in that
724 // new situation.
725 //
726 //------------------------------------------
727
728
729 CParse RightColumn = *m_Columns[ i ];
730 CParse LeftColumn = *m_Columns[i-1];
731 CParse LeftColumnOtherMorphemes = LeftColumn;
732
733 LeftColumn .RemovePiecesThatDoNotEnd ( ssSuffix );
734 LeftColumn .RemoveSuffixFromAllPieces ( ssSuffix );
735 LeftColumnOtherMorphemes
736 .RemovePiecesThatEnd ( ssSuffix );
737
738 RightColumn .PrefixToAllPieces2 ( ssSuffix );
739
740 double OldComplexity = m_Columns[i] ->ComputeComplexity() +
741 m_Columns[i-1]->ComputeComplexity();
742
743 double NewComplexity = RightColumn.ComputeComplexity() +
744 LeftColumn.ComputeComplexity();
745
746 if (LeftColumnOtherMorphemes.Size() > 0 )
747 {
748 NewComplexity += LeftColumnOtherMorphemes.ComputeComplexity() +
749 m_Columns[i]->ComputeComplexity();
750 }
751
752
753
754 if ( NewComplexity < OldComplexity )
755 {
756 if ( LeftColumnOtherMorphemes.Size() > 0 )
757 {
758 CTemplate TempTemplate ( *this );
759 *TempTemplate.GetColumn( i-1 ) = LeftColumnOtherMorphemes;
760 *TempTemplate.GetColumn( i ) = *m_Columns[ i ];
761
762 TempTemplates->AddTemplate ( &TempTemplate);
763 }
764
765 *m_Columns[i] = RightColumn;
766 *m_Columns[i-1] = LeftColumn;
767
768 }
769
770 continue;
771
772
773 }
774
775
776 return Return;
777
778 }
779
780 /*
781 void CTemplate::OutputForFile(ofstream& out )
782 {
783 int MaxSize = (int) m_Columns[0]->Size();
784 CString Space = " ";
785 for ( int i = 1; i < m_NumberOfColumns; i++)
786 {
787 if ( (int) m_Columns[i]->Size() > MaxSize )
788 {
789 MaxSize = m_Columns[i]->Size();
790 }
791 }
792 for (int row = 0; row < MaxSize; row++)
793 {
794 out << endl;
795 for (int col = 0; col < m_NumberOfColumns; col++)
796 {
797 out << setw(20);
798 if ( row < (int) m_Columns[col]->Size() )
799 {
800 out << m_Columns[col]->GetAt(row+1) << ' ';
801 }
802 else
803 {
804 out << '-';
805 }
806 }
807 }
808 out << endl << endl << "Complexity: "<< endl;
809 out << "From length: "<< base2log (m_NumberOfColumns) << endl;
810 for ( i = 0; i < m_NumberOfColumns; i++)
811 {
812 out << i << " " << m_Columns[i]->ComputeComplexity() << endl;
813 }
814 }
815
816 */
817
818 int CTemplate::GetNumberOfLetters()
819 {
820
821 int Total = 0;
822
823 for (int i = 0; i < m_NumberOfColumns; i++)
824 {
825 Total += m_Columns[i]->GetKeyLength();
826 }
827
828 return Total;
829 }
830
831 float CTemplate::ComputeComplexity() const
832 {
833 m_Complexity = 0;
834 m_Complexity += float ( base2log (m_NumberOfColumns) +
835 g_Lambda * m_NumberOfColumns * (m_NumberOfColumns - 1 ) / 2 );
836 Q_ASSERT (m_Complexity < 20 );
837 for (int i = 0; i < m_NumberOfColumns; i++)
838 {
839 m_Complexity += (float) m_Columns[ i ]->ComputeComplexity();
840 Q_ASSERT ( m_Complexity < 55000 );
841 }
842 Q_ASSERT ( m_Complexity > 0 );
843 Q_ASSERT ( m_Complexity < 55000 );
844
845
846
847 return m_Complexity;
848 }
849
850 float CTemplate::ComputeWordsTotalComplexity() const
851 {
852
853 CParse TempWords,
854 NewWords;
855 // first figure out the words:
856 if (m_NumberOfColumns < 1)
857 {
858 m_WordsTotalComplexity= 0;
859 //out << " No columns." ;
860 return m_WordsTotalComplexity;
861 }
862
863 CParse Words = *m_Columns[0];
864 for (int col = 1; col < m_NumberOfColumns; col++)
865 {
866 NewWords.ClearParse();
867 for (int row = 1; row <= (int) m_Columns[col]->Size(); row++)
868 {
869 TempWords = Words;
870 CStringSurrogate ss = m_Columns[col]->GetPiece(row);
871 if (ss == CStringSurrogate(QString("NULL").unicode(), 0, 4) )
872 {
873 // TempWords = Words;
874 } else
875 {
876 TempWords.SuffixToAllPieces2 ( m_Columns[col]->GetPiece(row) );
877 }
878 NewWords.Append (&TempWords);
879 }
880 Words = NewWords;
881
882 }
883
884 m_WordsTotalComplexity = (float) Words.ComputeComplexity();
885
886 Q_ASSERT (m_WordsTotalComplexity < 50000 );
887
888 return m_WordsTotalComplexity;
889
890 }
891
892
893
894 float CTemplate:: GetComplexity()
895 {
896 if (m_Complexity <= 0)
897 {
898 ComputeComplexity();
899 }
900 return m_Complexity;
901 }
902 float CTemplate:: GetWordsTotalComplexity()
903 {
904 if (m_WordsTotalComplexity <= 0)
905 {
906 ComputeWordsTotalComplexity();
907 }
908 return m_WordsTotalComplexity;
909 }
910
911
912
913 void CTemplate::GetWordsAndParses(StringToParse& OneWordAndParse)
914 {
915 int CurrentSize ;
916 int Column;
917 int row;
918 StringToParse TempWordsAndParses;
919 StringToParse SwapWordsAndParses;
920 StringToParse::Iterator StringToParseIt;
921 QString TheWord, OneWord, NewWord, DumpWord;
922 CParse* TheParse;
923 CParse* NewParse;
924 bool IsPreNull;
925
926
927 CurrentSize = (int) m_Columns[0]->Size();
928 for ( row= 0; row<CurrentSize; row++)
929 {
930 TheWord = m_Columns[0]->GetPiece(row+1).Display();
931 //Delete the "#"
932 TheWord = TheWord.right((TheWord.length()-1));
933 if ( TheWord.length() == 0)
934 {
935 TheWord = QString("NULL");
936 }
937
938 TheParse = new CParse(CStringSurrogate(TheWord));
939 TempWordsAndParses.insert(TheWord,TheParse);
940 }
941
942
943 for ( int i = 1; i < m_NumberOfColumns; i++)
944 {
945 Column = i;
946
947 CurrentSize = (int) m_Columns[Column]->Size();
948
949 for ( StringToParseIt = TempWordsAndParses.begin();StringToParseIt != TempWordsAndParses.end(); StringToParseIt++)
950 {
951 TheWord = StringToParseIt.key();
952 TheParse = StringToParseIt.data();
953
954 // It only works for first col;
955 if (TheWord == QString("NULL"))
956 {
957 IsPreNull = true;
958 }
959 else
960 {
961 IsPreNull = false;
962 }
963
964 for ( row= 0; row<CurrentSize; row++)
965 {
966 OneWord = m_Columns[Column]->GetPiece(row+1).Display();
967
968
969 if ( OneWord == QString("NULL"))
970 {
971 // Impossible We get "NULL" + "NULL"
972 NewWord = TheWord;
973 NewParse = new CParse(*TheParse);
974 }
975 else
976 {
977 if (! IsPreNull)
978 {
979 NewWord = TheWord + OneWord;
980 NewParse = new CParse(*TheParse);
981 NewParse->Append(CStringSurrogate(OneWord));
982 }
983 else
984 {
985 NewWord = OneWord;
986 NewParse = new CParse();
987 NewParse->Append(CStringSurrogate(OneWord));
988 }
989
990 }
991
992 if (! SwapWordsAndParses.contains(NewWord))
993 {
994 SwapWordsAndParses.insert(NewWord,NewParse);
995 }
996 else
997 {
998 QMessageBox::information( NULL, "debug","Impossible Here. Please Let Yu Hu Debug it!", "OK") ;
999 }
1000 }
1001
1002 delete TheParse;
1003
1004 }
1005
1006 TempWordsAndParses.clear();
1007
1008 for ( StringToParseIt = SwapWordsAndParses.begin();StringToParseIt != SwapWordsAndParses.end(); StringToParseIt++)
1009 {
1010 TheWord = StringToParseIt.key();
1011 TheParse = StringToParseIt.data();
1012
1013 TempWordsAndParses.insert(TheWord, TheParse);
1014 }
1015
1016 SwapWordsAndParses.clear();
1017
1018 }
1019
1020
1021
1022 for ( StringToParseIt = TempWordsAndParses.begin();StringToParseIt != TempWordsAndParses.end(); StringToParseIt++)
1023 {
1024 TheWord = StringToParseIt.key();
1025 TheParse = StringToParseIt.data();
1026 OneWordAndParse.insert(TheWord, TheParse);
1027 }
1028
1029
1030 }
1031
1032
1033
1034
1035 //////////////////
1036
1037 void CTemplate::AdjustMeByMovingCommonTailOrHead2(StringToFloat& MorphemeAndItsComplexity, int MaximumSizeOfEachColumn, int MaximumNumberOfMovingLetters, bool PrintChangedTemplates,\
1038 int Loopi, bool ShouldUseStickNess, StringToStringToFloat& GlobalStickNess, \
1039 float TotalGlobalStickNess, float TotalWords)
1040 {
1041
1042
1043
1044 int i, row;
1045 int backletteri, headletteri;
1046 QString TheMorpheme;
1047 bool PassColumnSizeTest = false;
1048 CParse* LeftColumn, *RightColumn, *OneColumn;
1049 bool changed = false;
1050 bool NoFurther;
1051 int MoveWhichColumn;
1052 int MoveL2RorR2L; // 0 --> left to right; 1 --> right to left
1053 int MoveHowmanyLetters = 0;
1054 float OneMorphemeComplexity, AlternativeMorphemeComplexity;
1055 QString CommonTail, CommonHead, MyTail, MyHead;
1056 QString Newmorpheme;
1057 float NewmorphemeComplexity, Diff, BestDiff, BestStickNessDiff;
1058 float StickNessDiff;
1059 float LeftColumnOldStickness, LeftColumnNewStickness;
1060 float RightColumnOldStickness, RightColumnNewStickness;
1061 int Tempi, Tempj;
1062 QString TempMorphemei, TempMorphemej;
1063 StringToFloat* oneCollection;
1064 float oneFloat;
1065 CParse PossibleNewColumn;
1066
1067
1068
1069 //Through each column
1070 for ( i = 0; i < m_NumberOfColumns -1; i++)
1071 {
1072
1073 if ( i == m_ModifiedColumn)
1074 {
1075 continue;
1076 }
1077
1078 LeftColumn = m_Columns[i];
1079 RightColumn = m_Columns[i+1];
1080
1081 if (( LeftColumn ->Size() <= MaximumSizeOfEachColumn) && (RightColumn ->Size() <= MaximumSizeOfEachColumn))
1082 {
1083 PassColumnSizeTest = true;
1084 }
1085 else
1086 {
1087 PassColumnSizeTest = false;
1088 continue;
1089 }
1090
1091
1092 LeftColumnOldStickness = 0;
1093 LeftColumnNewStickness = 0;
1094 RightColumnOldStickness = 0;
1095 RightColumnNewStickness = 0;
1096
1097 if ( ShouldUseStickNess )
1098 {
1099
1100 if ( LeftColumn ->Size() != 1)
1101 {
1102
1103 for ( Tempi = 1; Tempi < LeftColumn ->Size(); Tempi++)
1104 {
1105 TempMorphemei = LeftColumn ->GetPiece(Tempi).Display();
1106
1107 if ( i == 0)
1108 {
1109 TempMorphemei = TempMorphemei.right(TempMorphemei.length() -1);
1110 if ( TempMorphemei.length() ==0)
1111 {
1112 TempMorphemei = QString("NULL");
1113 }
1114
1115 }
1116
1117
1118 for ( Tempj = Tempi + 1; Tempj <= LeftColumn ->Size(); Tempj++)
1119 {
1120 TempMorphemej = LeftColumn ->GetPiece(Tempj).Display();
1121
1122 if ( i == 0)
1123 {
1124 TempMorphemej = TempMorphemej.right(TempMorphemej.length() -1);
1125 if ( TempMorphemej.length() ==0)
1126 {
1127 TempMorphemej = QString("NULL");
1128 }
1129
1130 }
1131
1132
1133 if ( TempMorphemei > TempMorphemej )
1134 {
1135 oneCollection = GlobalStickNess[TempMorphemei];
1136 oneFloat = (*oneCollection)[TempMorphemej];
1137 LeftColumnOldStickness += oneFloat;
1138 }
1139 else
1140 {
1141 oneCollection = GlobalStickNess[TempMorphemej];
1142 oneFloat = (*oneCollection)[TempMorphemei];
1143 LeftColumnOldStickness += oneFloat;
1144
1145 }
1146
1147 }
1148
1149
1150 }
1151
1152 }
1153
1154
1155 if ( RightColumn ->Size() != 1)
1156 {
1157
1158 for ( Tempi = 1; Tempi < RightColumn ->Size(); Tempi++)
1159 {
1160 TempMorphemei = RightColumn ->GetPiece(Tempi).Display();
1161
1162
1163 for ( Tempj = Tempi +1; Tempj <= RightColumn ->Size(); Tempj++)
1164 {
1165 TempMorphemej = RightColumn ->GetPiece(Tempj).Display();
1166
1167
1168 if ( TempMorphemei > TempMorphemej )
1169 {
1170 oneCollection = GlobalStickNess[TempMorphemei];
1171 oneFloat = (*oneCollection)[TempMorphemej];
1172 RightColumnOldStickness += oneFloat;
1173 }
1174 else
1175 {
1176 oneCollection = GlobalStickNess[TempMorphemej];
1177 oneFloat = (*oneCollection)[TempMorphemei];
1178 RightColumnOldStickness += oneFloat;
1179
1180 }
1181
1182 }
1183
1184
1185 }
1186
1187 }
1188
1189 } // ShouldUseStickNess, then get the LeftColumnOldStickness, RightColumnOldStickness
1190
1191
1192
1193 // Check LeftColumn
1194
1195 backletteri = 1;
1196 NoFurther = false;
1197
1198 while ( (backletteri <= MaximumNumberOfMovingLetters))
1199 {
1200
1201 LeftColumnNewStickness = 0;
1202 RightColumnNewStickness = 0;
1203
1204 // Need experiment on how to deal with "NULL" since "NULL" is too common
1205 if ( (static_cast <int> ( (LeftColumn ->Size() )) == 1) && (LeftColumn->GetPiece(1).Display().length()<=backletteri ))
1206 {
1207 NoFurther = true;
1208 break;
1209 }
1210
1211
1212 PossibleNewColumn.ClearParse();
1213
1214 // Check whether all words have backletteri tailletter in common
1215 for ( row =1; row <= LeftColumn ->Size(); row++)
1216 {
1217 TheMorpheme = LeftColumn->GetPiece(row).Display();
1218
1219
1220 if ( i == 0)
1221 {
1222 TheMorpheme = TheMorpheme.right(TheMorpheme.length() -1);
1223
1224 if ( TheMorpheme.length() ==0)
1225 {
1226 TheMorpheme = QString("NULL");
1227 }
1228
1229 }
1230
1231 if ( TheMorpheme == QString("NULL"))
1232 {
1233 NoFurther = true;
1234 break;
1235 }
1236
1237 if ( static_cast <int> ( TheMorpheme.length() ) < backletteri)
1238 {
1239 NoFurther = true;
1240 break;
1241 }
1242
1243 MyTail = TheMorpheme.right(backletteri);
1244
1245 if ( row == 1)
1246 {
1247 CommonTail = MyTail;
1248 continue;
1249 }
1250
1251 if ( MyTail != CommonTail)
1252 {
1253 NoFurther = true;
1254 break;
1255 }
1256 }
1257
1258 if ( NoFurther)
1259 {
1260 break;
1261 }
1262
1263 // Now Calculate the Alternative Complexity
1264 Diff = 0.0;
1265
1266 for ( row =1; row <= LeftColumn ->Size(); row++)
1267 {
1268 TheMorpheme = LeftColumn->GetPiece(row).Display();
1269
1270 if ( i == 0)
1271 {
1272 TheMorpheme = TheMorpheme.right(TheMorpheme.length() -1);
1273
1274 if ( TheMorpheme.length() ==0)
1275 {
1276 TheMorpheme = QString("NULL");
1277 }
1278
1279 }
1280
1281 if (!MorphemeAndItsComplexity.contains(TheMorpheme)) //, OneMorphemeComplexity))
1282 {
1283 QMessageBox::information(NULL, "Debug", "Can't find this Morpheme", "OK");
1284 return;
1285 }
1286
1287 OneMorphemeComplexity = MorphemeAndItsComplexity[TheMorpheme];
1288
1289 Newmorpheme = TheMorpheme.left(TheMorpheme.length() - backletteri);
1290
1291 if ( Newmorpheme.length() ==0)
1292 {
1293 Newmorpheme = QString("NULL");
1294 }
1295
1296 // Populate the possible Column
1297 PossibleNewColumn.Append(Newmorpheme);
1298
1299 if (!MorphemeAndItsComplexity.contains(Newmorpheme))//, AlternativeMorphemeComplexity))
1300 {
1301 NewmorphemeComplexity = -base2log(1.0/TotalWords); // Big Penalty
1302 }
1303 else
1304 {
1305 AlternativeMorphemeComplexity = MorphemeAndItsComplexity[Newmorpheme];
1306 NewmorphemeComplexity = AlternativeMorphemeComplexity;
1307 }
1308
1309 Diff += NewmorphemeComplexity - OneMorphemeComplexity;
1310
1311 }
1312
1313
1314
1315 // Compute the possible Column Stickness
1316 if ( ShouldUseStickNess )
1317 {
1318 if ( PossibleNewColumn.Size() > 1)
1319 {
1320
1321 for ( Tempi = 1; Tempi < PossibleNewColumn.Size(); Tempi++)
1322 {
1323 TempMorphemei = PossibleNewColumn.GetPiece(Tempi).Display();
1324
1325 for ( Tempj = Tempi + 1; Tempj <= PossibleNewColumn.Size(); Tempj++)
1326 {
1327 TempMorphemej = PossibleNewColumn.GetPiece(Tempj).Display();
1328
1329 if ( TempMorphemei > TempMorphemej )
1330 {
1331 if ( GlobalStickNess.contains(TempMorphemei))
1332 {
1333 oneCollection = GlobalStickNess[TempMorphemei];
1334
1335 if ( oneCollection ->contains(TempMorphemej))
1336 {
1337 oneFloat = (*oneCollection)[TempMorphemej];
1338 LeftColumnNewStickness += oneFloat;
1339 }
1340 else
1341 {
1342 LeftColumnNewStickness += -base2log(1.0/TotalGlobalStickNess); // Big Penalty
1343
1344 }
1345
1346 }
1347 else
1348 {
1349 LeftColumnNewStickness += -base2log(1.0/TotalGlobalStickNess); // Big Penalty
1350 }
1351 }
1352 else
1353 {
1354 if ( GlobalStickNess.contains(TempMorphemej))
1355 {
1356 oneCollection = GlobalStickNess[TempMorphemej];
1357 if ( oneCollection ->contains(TempMorphemei))
1358 {
1359 oneFloat = (*oneCollection)[TempMorphemei];
1360 LeftColumnNewStickness += oneFloat;
1361 }
1362 else
1363 {
1364 LeftColumnNewStickness += -base2log(1.0/TotalGlobalStickNess); // Big Penalty
1365
1366 }
1367
1368 }
1369 else
1370 {
1371 LeftColumnNewStickness += -base2log(1.0/TotalGlobalStickNess); // Big Penalty
1372 }
1373 }
1374
1375 }
1376
1377
1378 }
1379
1380 }
1381 else
1382 {
1383 LeftColumnNewStickness = 0.0;
1384 }
1385 }
1386
1387
1388 PossibleNewColumn.ClearParse();
1389 for ( row =1; row <= RightColumn ->Size(); row++)
1390 {
1391 TheMorpheme = RightColumn->GetPiece(row).Display();
1392
1393 if (!MorphemeAndItsComplexity.contains(TheMorpheme))
1394 {
1395 QMessageBox::information (NULL, "Debug", "Can't find this Morpheme", "OK");
1396 return;
1397 }
1398
1399 OneMorphemeComplexity = MorphemeAndItsComplexity[TheMorpheme];
1400
1401 if ( TheMorpheme == QString("NULL"))
1402 {
1403 Newmorpheme = CommonTail;
1404 }
1405 else
1406 {
1407 Newmorpheme = CommonTail + TheMorpheme;
1408 }
1409
1410 PossibleNewColumn.Append(Newmorpheme);
1411
1412 if (!MorphemeAndItsComplexity.contains(Newmorpheme)) //, ))
1413 {
1414 NewmorphemeComplexity = -base2log(1.0/TotalWords); // Big Penalty
1415 }
1416 else
1417 {
1418 AlternativeMorphemeComplexity = MorphemeAndItsComplexity[Newmorpheme];
1419 NewmorphemeComplexity = AlternativeMorphemeComplexity;
1420 }
1421
1422 Diff += NewmorphemeComplexity - OneMorphemeComplexity;
1423
1424 }
1425
1426
1427 // Compute the possible Column Stickness
1428 if ( ShouldUseStickNess )
1429 {
1430 if ( PossibleNewColumn.Size() > 1)
1431 {
1432
1433 for ( Tempi = 1; Tempi < PossibleNewColumn.Size(); Tempi++)
1434 {
1435 TempMorphemei = PossibleNewColumn.GetPiece(Tempi).Display();
1436
1437 for ( Tempj = Tempi + 1; Tempj <= PossibleNewColumn.Size(); Tempj++)
1438 {
1439 TempMorphemej = PossibleNewColumn.GetPiece(Tempj).Display();
1440
1441 if ( TempMorphemei > TempMorphemej )
1442 {
1443 if ( GlobalStickNess.contains(TempMorphemei))
1444 {
1445 oneCollection = GlobalStickNess[TempMorphemei];
1446 if ( oneCollection ->contains(TempMorphemej))
1447 {
1448 oneFloat = (*oneCollection)[TempMorphemej];
1449 RightColumnNewStickness += oneFloat;
1450 }
1451 else
1452 {
1453 RightColumnNewStickness += -base2log(1.0/TotalGlobalStickNess); // Big Penalty
1454
1455 }
1456
1457 }
1458 else
1459 {
1460 RightColumnNewStickness += -base2log(1.0/TotalGlobalStickNess); // Big Penalty
1461 }
1462 }
1463 else
1464 {
1465 if ( GlobalStickNess.contains(TempMorphemej))
1466 {
1467 oneCollection = GlobalStickNess[TempMorphemej];
1468 if ( oneCollection ->contains(TempMorphemei))
1469 {
1470 oneFloat = (*oneCollection)[TempMorphemei];
1471 RightColumnNewStickness += oneFloat;
1472 }
1473 else
1474 {
1475 RightColumnNewStickness += -base2log(1.0/TotalGlobalStickNess); // Big Penalty
1476
1477 }
1478
1479 }
1480 else
1481 {
1482 RightColumnNewStickness += -base2log(1.0/TotalGlobalStickNess); // Big Penalty
1483 }
1484 }
1485
1486 }
1487
1488
1489 }
1490
1491 }
1492 else
1493 {
1494 RightColumnNewStickness = 0.0;
1495 }
1496 }
1497
1498
1499
1500
1501 // Check whether Diff > 0, if so records these changes
1502
1503 if ( ShouldUseStickNess )
1504 {
1505
1506 StickNessDiff = ( RightColumnNewStickness + LeftColumnNewStickness) - (RightColumnOldStickness + LeftColumnOldStickness) ;
1507
1508 Diff += StickNessDiff;
1509
1510 }
1511
1512
1513 if (Diff < 0)
1514 {
1515 if (changed)
1516 {
1517 if ( Diff < BestDiff)
1518 {
1519 BestDiff = Diff;
1520 BestStickNessDiff = StickNessDiff;
1521 MoveWhichColumn = i;
1522 MoveL2RorR2L = 0;
1523 MoveHowmanyLetters = backletteri;
1524 // Always remember the left one column
1525 m_ModifiedColumn = i;
1526 }
1527
1528 }
1529 else
1530 {
1531 changed = true;
1532 BestDiff = Diff;
1533 MoveWhichColumn = i;
1534 MoveL2RorR2L = 0;
1535 MoveHowmanyLetters = backletteri;
1536 // Always remember the left one column
1537 m_ModifiedColumn = i;
1538 }
1539
1540 }
1541
1542 backletteri++;
1543 }
1544
1545
1546
1547 // Check RightColumn
1548
1549 headletteri = 1;
1550 NoFurther = false;
1551
1552 while ( (headletteri <= MaximumNumberOfMovingLetters))
1553 {
1554
1555 LeftColumnNewStickness = 0;
1556 RightColumnNewStickness = 0;
1557
1558 PossibleNewColumn.ClearParse();
1559
1560 // Need experiment on how to deal with "NULL" since "NULL" is too common
1561 if ( (static_cast <int> ((RightColumn ->Size())) == 1) && (RightColumn->GetPiece(1).Display().length()<=headletteri ))
1562 {
1563 NoFurther = true;
1564 break;
1565 }
1566
1567 // Check whether all words have headletteri headletter in common
1568 for ( row =1; row <= RightColumn ->Size(); row++)
1569 {
1570 TheMorpheme = RightColumn->GetPiece(row).Display();
1571
1572 if ( TheMorpheme == QString("NULL"))
1573 {
1574 NoFurther = true;
1575 break;
1576 }
1577
1578 if ( static_cast <int> ( TheMorpheme.length() ) < headletteri)
1579 {
1580 NoFurther = true;
1581 break;
1582 }
1583
1584 MyHead = TheMorpheme.left(headletteri);
1585
1586 if ( row == 1)
1587 {
1588 CommonHead = MyHead;
1589 continue;
1590 }
1591
1592 if ( MyHead != CommonHead)
1593 {
1594 NoFurther = true;
1595 break;
1596 }
1597 }
1598
1599 if ( NoFurther)
1600 {
1601 break;
1602 }
1603
1604 // Now Calculate the Alternative Complexity
1605 Diff = 0.0;
1606
1607 for ( row =1; row <= RightColumn ->Size(); row++)
1608 {
1609 TheMorpheme = RightColumn->GetPiece(row).Display();
1610
1611 if (!MorphemeAndItsComplexity.contains(TheMorpheme))
1612 {
1613 QMessageBox::information (NULL, "Debug", "Can't find this Morpheme", "OK");
1614 return;
1615 }
1616
1617
1618 OneMorphemeComplexity = MorphemeAndItsComplexity[TheMorpheme];
1619 Newmorpheme = TheMorpheme.right(TheMorpheme.length() - headletteri);
1620
1621 if ( Newmorpheme.length() ==0)
1622 {
1623 Newmorpheme = QString("NULL");
1624 }
1625
1626 PossibleNewColumn.Append(Newmorpheme);
1627
1628 if (!MorphemeAndItsComplexity.contains(Newmorpheme))
1629 {
1630 NewmorphemeComplexity = -base2log(1.0/TotalWords); // Big Penalty
1631 }
1632 else
1633 {
1634 AlternativeMorphemeComplexity = MorphemeAndItsComplexity[Newmorpheme];
1635 NewmorphemeComplexity = AlternativeMorphemeComplexity;
1636 }
1637
1638 Diff += NewmorphemeComplexity - OneMorphemeComplexity;
1639
1640 }
1641
1642 // Compute the possible Column Stickness
1643 if ( ShouldUseStickNess )
1644 {
1645 if ( PossibleNewColumn.Size() > 1)
1646 {
1647
1648 for ( Tempi = 1; Tempi < PossibleNewColumn.Size(); Tempi++)
1649 {
1650 TempMorphemei = PossibleNewColumn.GetPiece(Tempi).Display();
1651
1652 for ( Tempj = Tempi + 1; Tempj <= PossibleNewColumn.Size(); Tempj++)
1653 {
1654 TempMorphemej = PossibleNewColumn.GetPiece(Tempj).Display();
1655
1656 if ( TempMorphemei > TempMorphemej )
1657 {
1658 if ( GlobalStickNess.contains(TempMorphemei))
1659 {
1660 oneCollection = GlobalStickNess[TempMorphemei];
1661 if ( oneCollection ->contains(TempMorphemej))
1662 {
1663 oneFloat = (*oneCollection)[TempMorphemej];
1664 RightColumnNewStickness += oneFloat;
1665 }
1666 else
1667 {
1668 RightColumnNewStickness += -base2log(1.0/TotalGlobalStickNess); // Big Penalty
1669
1670 }
1671
1672 }
1673 else
1674 {
1675 RightColumnNewStickness += -base2log(1.0/TotalGlobalStickNess); // Big Penalty
1676 }
1677 }
1678 else
1679 {
1680 if ( GlobalStickNess.contains(TempMorphemej))
1681 {
1682 oneCollection = GlobalStickNess[TempMorphemej];
1683 if ( oneCollection ->contains(TempMorphemei))
1684 {
1685 oneFloat = (*oneCollection)[TempMorphemei];
1686 RightColumnNewStickness += oneFloat;
1687 }
1688 else
1689 {
1690 RightColumnNewStickness += -base2log(1.0/TotalGlobalStickNess); // Big Penalty
1691
1692 }
1693
1694 }
1695 else
1696 {
1697 RightColumnNewStickness += -base2log(1.0/TotalGlobalStickNess); // Big Penalty
1698 }
1699 }
1700
1701 }
1702
1703
1704 }
1705
1706 }
1707 else
1708 {
1709 RightColumnNewStickness = 0.0;
1710 }
1711 }
1712
1713 PossibleNewColumn.ClearParse();
1714 for ( row =1; row <= LeftColumn ->Size(); row++)
1715 {
1716 TheMorpheme = LeftColumn->GetPiece(row).Display();
1717
1718 if ( i == 0)
1719 {
1720 TheMorpheme = TheMorpheme.right(TheMorpheme.length() -1);
1721 if ( TheMorpheme.length() ==0)
1722 {
1723 TheMorpheme = QString("NULL");
1724 }
1725
1726 }
1727
1728 if (!MorphemeAndItsComplexity.contains(TheMorpheme))
1729 {
1730 QMessageBox::information (NULL, "Debug", "Can't find this Morpheme", "OK");
1731 return;
1732 }
1733
1734
1735 OneMorphemeComplexity = MorphemeAndItsComplexity[TheMorpheme];
1736
1737 if ( TheMorpheme == QString("NULL"))
1738 {
1739 Newmorpheme = CommonHead;
1740 }
1741 else
1742 {
1743 Newmorpheme = TheMorpheme + CommonHead;
1744 }
1745
1746 PossibleNewColumn.Append(Newmorpheme);
1747
1748 if (!MorphemeAndItsComplexity.contains(Newmorpheme))
1749 {
1750 NewmorphemeComplexity = -base2log(1.0/TotalWords); // Big Penalty
1751 }
1752 else
1753 {
1754 AlternativeMorphemeComplexity = MorphemeAndItsComplexity[Newmorpheme];
1755 NewmorphemeComplexity = AlternativeMorphemeComplexity;
1756 }
1757
1758 Diff += NewmorphemeComplexity - OneMorphemeComplexity;
1759
1760 }
1761
1762 // Compute the possible Column Stickness
1763 if ( ShouldUseStickNess )
1764 {
1765 if ( PossibleNewColumn.Size() > 1)
1766 {
1767
1768 for ( Tempi = 1; Tempi < PossibleNewColumn.Size(); Tempi++)
1769 {
1770 TempMorphemei = PossibleNewColumn.GetPiece(Tempi).Display();
1771
1772 for ( Tempj = Tempi + 1; Tempj <= PossibleNewColumn.Size(); Tempj++)
1773 {
1774 TempMorphemej = PossibleNewColumn.GetPiece(Tempj).Display();
1775
1776 if ( TempMorphemei > TempMorphemej )
1777 {
1778 if ( GlobalStickNess.contains(TempMorphemei))
1779 {
1780 oneCollection = GlobalStickNess[TempMorphemei];
1781 if ( oneCollection ->contains(TempMorphemej))
1782 {
1783 oneFloat = (*oneCollection)[TempMorphemej];
1784 LeftColumnNewStickness += oneFloat;
1785 }
1786 else
1787 {
1788 LeftColumnNewStickness += -base2log(1.0/TotalGlobalStickNess); // Big Penalty
1789
1790 }
1791
1792 }
1793 else
1794 {
1795 LeftColumnNewStickness += -base2log(1.0/TotalGlobalStickNess); // Big Penalty
1796 }
1797 }
1798 else
1799 {
1800 if ( GlobalStickNess.contains(TempMorphemej))
1801 {
1802 oneCollection = GlobalStickNess[TempMorphemej];
1803 if ( oneCollection ->contains(TempMorphemei))
1804 {
1805 oneFloat = (*oneCollection)[TempMorphemei];
1806 LeftColumnNewStickness += oneFloat;
1807 }
1808 else
1809 {
1810 LeftColumnNewStickness += -base2log(1.0/TotalGlobalStickNess); // Big Penalty
1811
1812 }
1813
1814 }
1815 else
1816 {
1817 LeftColumnNewStickness += -base2log(1.0/TotalGlobalStickNess); // Big Penalty
1818 }
1819 }
1820
1821 }
1822
1823
1824 }
1825
1826 }
1827 else
1828 {
1829 LeftColumnNewStickness = 0.0;
1830 }
1831 }
1832
1833
1834 // Check whether Diff > 0, if so records these changes
1835
1836 if ( ShouldUseStickNess )
1837 {
1838
1839 StickNessDiff = ( RightColumnNewStickness + LeftColumnNewStickness) - (RightColumnOldStickness + LeftColumnOldStickness) ;
1840
1841 Diff += StickNessDiff;
1842
1843 }
1844
1845 if (Diff < 0)
1846 {
1847 if (changed)
1848 {
1849 if ( Diff < BestDiff)
1850 {
1851 BestDiff = Diff;
1852 BestStickNessDiff = StickNessDiff;
1853 MoveWhichColumn = i+1;
1854 MoveL2RorR2L = 1;
1855 MoveHowmanyLetters = headletteri;
1856
1857 // Always remember the left one column
1858 m_ModifiedColumn = i;
1859 }
1860
1861 }
1862 else
1863 {
1864 changed = true;
1865 BestDiff = Diff;
1866 MoveWhichColumn = i+1;
1867 MoveL2RorR2L = 1;
1868 MoveHowmanyLetters = headletteri;
1869
1870 // Always remember the left one column
1871 m_ModifiedColumn = i;
1872 }
1873
1874 }
1875
1876 headletteri++;
1877 }
1878 }
1879
1880
1881
1882 // Now, we can know whether we adjust this template
1883
1884
1885 // int MaximumOutputMorphemeInOneColumn = 8;
1886 QString DisplayOfOneColumn;
1887
1888
1889 if ( changed )
1890 {
1891 // Print Original Templates
1892 if (PrintChangedTemplates)
1893 {
1894
1895 QFile file( "AdjustedTemplates.txt" );
1896
1897 if ( !file.open( QIODevice::WriteOnly | QIODevice::Append ) )
1898 {
1899 QMessageBox::information(NULL, "Error", "Can't Open the file!", "OK");
1900 return;
1901 }
1902
1903 Q3TextStream outf( &file );
1904
1905 //outf.open ("AdjustedTemplates.txt", ofstream::out | ofstream::app);
1906
1907 outf << "***********"<<Loopi<<"************" <<endl;
1908 outf << " Original Template:" << endl;
1909 outf << " ";
1910
1911 for ( i = 0; i < m_NumberOfColumns; i++)
1912 {
1913 OneColumn = m_Columns[i];
1914
1915 DisplayOfOneColumn = QString("{ ");
1916 for (int j = 1; j <= OneColumn ->Size();j++)
1917 {
1918 DisplayOfOneColumn += OneColumn ->GetPiece(j).Display();
1919 if ( j != OneColumn ->Size())
1920 {
1921 DisplayOfOneColumn += QString(" , ");
1922 }
1923 }
1924
1925 DisplayOfOneColumn += QString(" }");
1926 outf << DisplayOfOneColumn;
1927 if ( i != m_NumberOfColumns -1)
1928 {
1929 outf << " --> ";
1930 }
1931 }
1932
1933 outf << endl;
1934
1935 file.close();
1936 //outf.close();
1937 }
1938
1939
1940 // Modify this tempalte based on former bestdiff record
1941 CParse TempParse;
1942 QString MovedSubString;
1943 CStringSurrogate TempCSS;
1944
1945 if ( MoveL2RorR2L == 0)
1946 {
1947 // when letters are moved from left to right
1948 LeftColumn = m_Columns[MoveWhichColumn];
1949 RightColumn = m_Columns[MoveWhichColumn+1];
1950 MovedSubString = LeftColumn ->GetPiece(1).Display().right(MoveHowmanyLetters);
1951
1952 // Modify left column
1953 TempParse.ClearParse();
1954 for ( row =1; row <= LeftColumn ->Size(); row++)
1955 {
1956 TheMorpheme = LeftColumn->GetPiece(row).Display();
1957
1958
1959 if ( MoveWhichColumn ==0)
1960 {
1961 TheMorpheme = TheMorpheme.right(TheMorpheme.length() -1);
1962
1963 if ( TheMorpheme.length() ==0)
1964 {
1965 QMessageBox::information (NULL, "Debug", "Impossible Here!", "OK");
1966 TheMorpheme = QString("NULL");
1967 }
1968
1969 }
1970
1971
1972 Newmorpheme = TheMorpheme.left(TheMorpheme.length() - MoveHowmanyLetters);
1973
1974 if ( Newmorpheme.length() ==0)
1975 {
1976 Newmorpheme = QString("NULL");
1977 }
1978
1979
1980 TempCSS = CStringSurrogate(Newmorpheme);
1981 if ( !TempParse.Contains(TempCSS))
1982 {
1983 TempParse.Append(Newmorpheme);
1984 }
1985
1986 }
1987
1988 if ( TempParse.Size() ==0)
1989 {
1990 QMessageBox::information(NULL, "Debug", "TempParse Size should not be 0", "OK");
1991 return;
1992 }
1993
1994 // If this is the first column, we put "#" back
1995 if ( MoveWhichColumn ==0)
1996 {
1997 CParse SwapParse;
1998 int Swapi;
1999 QString Swappiece;
2000
2001 for ( Swapi = 1; Swapi <= TempParse.Size(); Swapi++)
2002 {
2003 Swappiece = TempParse.GetPiece(Swapi).Display();
2004
2005 if ( Swappiece == QString("NULL"))
2006 {
2007 Swappiece = QString("#");
2008 }
2009 else
2010 {
2011 Swappiece = QString("#") + Swappiece;
2012 }
2013
2014 SwapParse.Append(Swappiece);
2015 }
2016
2017 TempParse = SwapParse ;
2018 }
2019
2020 (*LeftColumn) = TempParse;
2021
2022
2023 // Modify right column
2024 TempParse.ClearParse();
2025 for ( row =1; row <= RightColumn ->Size(); row++)
2026 {
2027 TheMorpheme = RightColumn->GetPiece(row).Display();
2028
2029 if ( TheMorpheme == QString("NULL"))
2030 {
2031 Newmorpheme = MovedSubString;
2032 }
2033 else
2034 {
2035 Newmorpheme = MovedSubString + TheMorpheme;
2036 }
2037
2038 TempCSS = CStringSurrogate(Newmorpheme);
2039 if ( !TempParse.Contains(TempCSS))
2040 {
2041 TempParse.Append(Newmorpheme);
2042 }
2043
2044
2045 }
2046
2047 if ( TempParse.Size() ==0)
2048 {
2049 QMessageBox::information(NULL, "Debug", "TempParse Size should not be 0", "OK");
2050 return;
2051 }
2052
2053 (*RightColumn) = TempParse;
2054
2055 }
2056 else
2057 {
2058 // when letters are moved from right to left
2059 LeftColumn = m_Columns[MoveWhichColumn-1];
2060 RightColumn = m_Columns[MoveWhichColumn];
2061 MovedSubString = RightColumn ->GetPiece(1).Display().left(MoveHowmanyLetters);
2062
2063 // Modify right column
2064 TempParse.ClearParse();
2065 for ( row =1; row <= RightColumn ->Size(); row++)
2066 {
2067 TheMorpheme = RightColumn->GetPiece(row).Display();
2068 Newmorpheme = TheMorpheme.right(TheMorpheme.length() - MoveHowmanyLetters);
2069 if ( Newmorpheme.length() ==0)
2070 {
2071 Newmorpheme = QString("NULL");
2072 }
2073
2074 TempCSS = CStringSurrogate(Newmorpheme);
2075 if ( !TempParse.Contains(TempCSS))
2076 {
2077 TempParse.Append(Newmorpheme);
2078 }
2079
2080 }
2081
2082 if ( TempParse.Size() ==0)
2083 {
2084 QMessageBox::information(NULL, "Debug", "TempParse Size should not be 0", "OK");
2085 return;
2086 }
2087
2088 (*RightColumn) = TempParse;
2089
2090
2091 // Modify left column
2092 TempParse.ClearParse();
2093 for ( row =1; row <= LeftColumn ->Size(); row++)
2094 {
2095 TheMorpheme = LeftColumn->GetPiece(row).Display();
2096
2097 if ( MoveWhichColumn == 1)
2098 {
2099 TheMorpheme = TheMorpheme.right(TheMorpheme.length() -1);
2100 if ( TheMorpheme.length() ==0)
2101 {
2102 TheMorpheme = QString("NULL");
2103 }
2104
2105 }
2106
2107 if ( TheMorpheme == QString("NULL"))
2108 {
2109 Newmorpheme = MovedSubString;
2110 }
2111 else
2112 {
2113 Newmorpheme = TheMorpheme + MovedSubString;
2114 }
2115
2116 TempCSS = CStringSurrogate(Newmorpheme);
2117 if ( !TempParse.Contains(TempCSS))
2118 {
2119 TempParse.Append(Newmorpheme);
2120 }
2121
2122 }
2123
2124 if ( TempParse.Size() ==0)
2125 {
2126 QMessageBox::information(NULL, "Debug", "TempParse Size should not be 0", "OK");
2127 return;
2128 }
2129
2130 // If this is the first column, we put "#" back
2131 if ( MoveWhichColumn == 1)
2132 {
2133 CParse SwapParse;
2134 int Swapi;
2135 QString Swappiece;
2136
2137 for ( Swapi = 1; Swapi <= TempParse.Size(); Swapi++)
2138 {
2139 Swappiece = TempParse.GetPiece(Swapi).Display();
2140
2141 if ( Swappiece == QString("NULL"))
2142 {
2143 Swappiece = QString("#");
2144 }
2145 else
2146 {
2147 Swappiece = QString("#") + Swappiece;
2148 }
2149
2150 SwapParse.Append(Swappiece);
2151 }
2152
2153 TempParse = SwapParse ;
2154 }
2155
2156 (*LeftColumn) = TempParse;
2157
2158 }
2159
2160
2161 // Check through this template, delete those column, which only has "NULL" or "#"
2162 int Newi;
2163 bool DeleteFirstColumn;
2164
2165
2166 DeleteFirstColumn = false;
2167 Newi = 0;
2168 for ( i = 0; i < m_NumberOfColumns; i++)
2169 {
2170 OneColumn = m_Columns[i];
2171
2172 if (( OneColumn ->Size() ==1 )&&(OneColumn ->GetPiece(1).Display() == QString("#")))
2173 {
2174 DeleteFirstColumn = true;
2175 continue;
2176 }
2177
2178 if ( ( OneColumn ->Size() ==1 )&&(OneColumn ->GetPiece(1).Display() == QString("NULL")))
2179 {
2180 continue;
2181 }
2182
2183 if ( DeleteFirstColumn && (Newi == 0))
2184 {
2185 CParse SwapParse;
2186 int Swapi;
2187 QString Swappiece;
2188
2189 for ( Swapi = 1; Swapi <= OneColumn ->Size(); Swapi++)
2190 {
2191 Swappiece = OneColumn ->GetPiece(Swapi).Display();
2192
2193 if ( Swappiece == QString("NULL"))
2194 {
2195 Swappiece = QString("#");
2196 }
2197 else
2198 {
2199 Swappiece = QString("#") + Swappiece;
2200 }
2201
2202 SwapParse.Append(Swappiece);
2203 }
2204
2205 (*OneColumn) = SwapParse ;
2206 }
2207
2208
2209 m_Columns[Newi] = OneColumn;
2210 Newi++;
2211 }
2212
2213 m_NumberOfColumns = Newi;
2214
2215
2216 // Print Result Templates
2217 if (PrintChangedTemplates)
2218 {
2219 QFile file( "AdjustedTemplates.txt" );
2220
2221 if ( !file.open( QIODevice::WriteOnly | QIODevice::Append) )
2222 {
2223 QMessageBox::information(NULL, "Error", "Can't Open the file!", "OK");
2224 return;
2225 }
2226
2227 Q3TextStream outf( &file );
2228
2229 //outf.open ("AdjustedTemplates.txt", ofstream::out | ofstream::app);
2230
2231 outf << " Adjusted Template:" << endl;
2232 outf << " ";
2233
2234 for ( i = 0; i < m_NumberOfColumns; i++)
2235 {
2236 OneColumn = m_Columns[i];
2237
2238 DisplayOfOneColumn = QString("{ ");
2239 for (int j = 1; j <= OneColumn ->Size(); j++)
2240 {
2241 DisplayOfOneColumn += OneColumn ->GetPiece(j).Display();
2242 if ( j != OneColumn ->Size())
2243 {
2244 DisplayOfOneColumn += QString(" , ");
2245 }
2246 }
2247
2248 DisplayOfOneColumn += QString(" }");
2249 outf << DisplayOfOneColumn;
2250 if ( i != m_NumberOfColumns -1)
2251 {
2252 outf << " --> ";
2253 }
2254 }
2255
2256 outf << endl;
2257
2258 outf << "The Diff in Pointer is " << BestDiff << ", The StickNess Diff is " << BestStickNessDiff <<".";
2259
2260 outf << endl;
2261 outf << endl;
2262 outf << endl;
2263
2264 file.close();
2265
2266 }
2267
2268
2269 }
2270 else
2271 {
2272 QFile file( "AdjustedTemplates.txt" );
2273
2274 if ( !file.open( QIODevice::WriteOnly | QIODevice::Append ) )
2275 {
2276 QMessageBox::information(NULL, "Error", "Can't Open the file!", "OK");
2277 return;
2278 }
2279
2280 Q3TextStream outf( &file );
2281
2282 outf<< "No Change for this template !" <<endl;
2283
2284 file.close();
2285
2286
2287 }
2288
2289 }
2290
2291 namespace {
2292 QString omit_first_letter(const QString& morpheme)
2293 {
2294 Q_ASSERT(morpheme != TheStringNULL);
2295 Q_ASSERT(morpheme.size() >= 1);
2296
2297 if (morpheme.size() == 1)
2298 return TheStringNULL;
2299 else
2300 return morpheme.mid(1);
2301 }
2302 }
2303
2304 void CTemplate::AdjustMeAfterAbsorb1(QMap<QString, int>& AbsorbedWords,
2305 bool Conservative, int MiniMumPrefixOrSuffixNeedToBeAbsorbed, int Loopi)
2306 {
2307 const bool OpenPrint = true;
2308
2309 int StemSize = 0;
2310 int StemColumn = -1;
2311 for (int i = 0; i < m_NumberOfColumns; ++i) {
2312 CParse& column = *m_Columns[i];
2313
2314 if (column.Size() > StemSize) {
2315 StemSize = column.Size();
2316 StemColumn = i;
2317 }
2318 }
2319
2320 Q_ASSERT(StemSize >= 0);
2321 Q_ASSERT(StemColumn >= 0 && StemColumn < m_NumberOfColumns);
2322 Q_ASSERT(m_NumberOfColumns == 2 || m_NumberOfColumns == 3);
2323
2324 // We don't consider the case where Stem is in middle
2325 if (m_NumberOfColumns == 3 && StemColumn == 1)
2326 return;
2327
2328 Q_ASSERT(StemColumn == 0 || StemColumn == 1 || StemColumn == 2);
2329
2330 if (StemColumn == 1)
2331 // Already returned if number of columns == 3.
2332 Q_ASSERT(m_NumberOfColumns == 2);
2333
2334 if (StemColumn == 2)
2335 // it only can be 3-column
2336 Q_ASSERT(m_NumberOfColumns == 3);
2337
2338 // values are all 1. XXX. use std::set.
2339 QMap<QString, int> affix_set;
2340
2341 if (m_NumberOfColumns == 3) {
2342 Q_ASSERT(StemColumn == 0);
2343 GetWordsFromTwoColums(affix_set, 1);
2344 } else {
2345 Q_ASSERT(m_NumberOfColumns == 2);
2346
2347 CParse& column = *m_Columns[1 - StemColumn];
2348 for (int row = 1; row <= column.Size(); ++row) {
2349 QString affix = column.GetPiece(row).Display();
2350 if (StemColumn == 1)
2351 affix = omit_first_letter(affix);
2352
2353 affix_set[affix] = 1;
2354 }
2355 }
2356
2357 // With these affixes, check the stem column
2358 CParse& column = *m_Columns[StemColumn];
2359 CParse TempParse;
2360 QMap<QString, int> AbsorbedStems;
2361 int AbsorbedStemSize = 0;
2362 for (int row = 1; row <= column.Size(); ++row) {
2363 QString stem = column.GetPiece(row).Display();
2364 if (StemColumn == 0)
2365 stem = omit_first_letter(stem);
2366
2367 int FitHowManyPrefixOrSuffix = 0;
2368 for (QMap<QString, int>::const_iterator iter =
2369 affix_set.constBegin();
2370 iter != affix_set.constEnd();
2371 ++iter) {
2372 QString affix = iter.key();
2373 Q_ASSERT(iter.value() == 1);
2374
2375 QString MakeUpWord;
2376 if (stem != TheStringNULL)
2377 MakeUpWord.append(stem);
2378
2379 if (affix != TheStringNULL) {
2380 if (StemColumn == 0) {
2381 MakeUpWord.append(affix);
2382 } else {
2383 Q_ASSERT(StemColumn ==
2384 m_NumberOfColumns - 1);
2385 MakeUpWord.prepend(affix);
2386 }
2387 }
2388
2389 if (AbsorbedWords.contains(MakeUpWord))
2390 ++FitHowManyPrefixOrSuffix;
2391 }
2392
2393 int CheckLimit = affix_set.size();
2394 if (Conservative == 0)
2395 CheckLimit = std::min(CheckLimit,
2396 MiniMumPrefixOrSuffixNeedToBeAbsorbed);
2397
2398 if (FitHowManyPrefixOrSuffix < CheckLimit) {
2399 TempParse.Append(CStringSurrogate(stem));
2400 } else {
2401 ++AbsorbedStemSize;
2402 AbsorbedStems[stem] = 1;
2403 }
2404 }
2405
2406 if (AbsorbedStemSize == StemSize)
2407 m_IsDeleted = true;
2408 else if (AbsorbedStemSize > 0)
2409 std::swap(*(m_Columns[StemColumn]), TempParse);
2410 else
2411 return;
2412
2413 if (!OpenPrint)
2414 return;
2415
2416 QFile file("Absorb1.txt");
2417 if (!file.open(QIODevice::WriteOnly | QIODevice::Append)) {
2418 QMessageBox::information(0, "Error",
2419 "Can't Open the file!", "OK");
2420 return;
2421 }
2422 QTextStream outf(&file);
2423
2424 outf << "*************" << Loopi << "**************" << endl;
2425 if (AbsorbedStemSize == StemSize) {
2426 outf << "\tDelete This Template:" << endl;
2427 } else {
2428 Q_ASSERT(AbsorbedStemSize > 0);
2429 // Took some stems away from one template
2430 outf << "\tAbsorb Some Stems From This Template:" <<
2431 endl;
2432 }
2433 outf << "\t\t";
2434
2435 for (int i = 0; i < m_NumberOfColumns; ++i) {
2436 CParse& column = i != StemColumn
2437 ? *m_Columns[i]
2438 : TempParse; // old *m_Columns[i]
2439
2440 outf << "{ ";
2441 for (int j = 1; j <= column.Size(); ++j) {
2442 outf << column.GetPiece(j).Display();
2443 if (j == column.Size())
2444 break;
2445 outf << " , ";
2446 }
2447 outf << " }";
2448
2449 if (i == m_NumberOfColumns - 1)
2450 break;
2451 outf << " --> ";
2452 }
2453 outf << endl;
2454
2455 if (AbsorbedStemSize != StemSize) {
2456 Q_ASSERT(AbsorbedStemSize > 0);
2457 outf << "Absorbed Stems: ";
2458
2459 outf << "{ ";
2460 if (!AbsorbedStems.isEmpty()) {
2461 QMap<QString, int>::const_iterator iter =
2462 AbsorbedStems.constBegin();
2463
2464 outf << iter.key(); // first stem
2465 Q_ASSERT(iter.value() == 1);
2466
2467 for (++iter; iter != AbsorbedStems.constEnd();
2468 ++iter) {
2469 outf << " , " << iter.key();
2470 Q_ASSERT(iter.value() == 1);
2471 }
2472 }
2473 outf << " }" << endl;
2474 }
2475 outf << endl;
2476 }
2477
2478 void CTemplate::GetWordsFromTwoColums(StringToInt& AllPrefixes, int StartColumn)
2479 {
2480
2481 int CurrentSize ;
2482 int Column;
2483 int row;
2484 QString TheWord, OneWord, NewWord, DumpWord;
2485 StringToInt::iterator StringToIntIt;
2486 bool IsPreNull;
2487 StringToInt TempWords;
2488
2489
2490 if ( GetNumberOfColumns() < StartColumn +2)
2491 {
2492 QMessageBox::information(NULL, "Error", "Impossible Here So far. Please Let Yu Hu Debug it!", "OK") ;
2493 return;
2494 }
2495
2496
2497 AllPrefixes.clear();
2498
2499 CurrentSize = (int) m_Columns[StartColumn]->Size();
2500 for ( row= 0; row<CurrentSize; row++)
2501 {
2502 TheWord = m_Columns[StartColumn]->GetPiece(row+1).Display();
2503
2504 if ( StartColumn == 0)
2505 {
2506 TheWord = TheWord.right((TheWord.length()-1));
2507 if ( TheWord.length() ==0)
2508 {
2509 TheWord = QString("NULL");
2510 }
2511 }
2512
2513 TempWords.insert(TheWord,1);
2514 }
2515
2516
2517 Column = StartColumn + 1;
2518
2519 CurrentSize = (int) m_Columns[Column]->Size();
2520
2521 for ( StringToIntIt = TempWords.begin(); StringToIntIt != TempWords.end(); StringToIntIt++)
2522 {
2523 TheWord = StringToIntIt.key();
2524
2525 // It only works for first col;
2526 if (TheWord == QString("NULL"))
2527 {
2528 IsPreNull = true;
2529 }
2530 else
2531 {
2532 IsPreNull = false;
2533 }
2534
2535 for ( row= 0; row<CurrentSize; row++)
2536 {
2537 OneWord = m_Columns[Column]->GetPiece(row+1).Display();
2538
2539 if ( OneWord == QString("NULL"))
2540 {
2541 // possible We get "NULL" + "NULL"
2542 NewWord = TheWord;
2543 }
2544 else
2545 {
2546 if (! IsPreNull)
2547 {
2548 NewWord = TheWord + OneWord;
2549 }
2550 else
2551 {
2552 NewWord = OneWord;
2553 }
2554
2555 }
2556
2557 if (! AllPrefixes.contains(NewWord))
2558 {
2559 AllPrefixes.insert(NewWord,1);
2560 }
2561 else
2562 {
2563 QMessageBox::information(NULL, "Error" ,"Impossible Here. Please Let Yu Hu Debug it!", "OK") ;
2564 }
2565 }
2566
2567 }
2568
2569
2570 }
2571
2572
2573
2574
2575
2576 void CTemplate::FindMorphemePrefixOrSuffixWithParadigmaticGraph(int Loopi, int MaximumSizeOfStemColumn, StringToFloat& GlobalNodeStickNess2, StringToStringToFloat& TotalGlobalStickNess2, float GlobalStickNess2, float TotalWord2)
2577 {
2578
2579 CParse* oneColumn;
2580 int i, itemi, itemj;
2581 QString Morphemei, Morphemej;
2582 bool IsMePrefixOfOthers, IsMeSuffixOfOthers;
2583 QString ThePrefixOrSuffix;
2584 bool FoundAndModified;
2585 bool HasEverChanged;
2586 CTemplate* BackUpTemplate;
2587 CStringSurrogate TempCSS, TempCSS1;
2588 CStringSurrogate NULLCSS;
2589
2590
2591 NULLCSS = CStringSurrogate(QString("NULL"));
2592
2593
2594
2595 BackUpTemplate = new CTemplate(*this);
2596
2597
2598 HasEverChanged = false;
2599 FoundAndModified = true;
2600
2601 while( FoundAndModified )
2602 {
2603
2604 FoundAndModified = false;
2605 for ( i = 0; i < m_NumberOfColumns; i++)
2606 {
2607 oneColumn = m_Columns[i];
2608
2609 if ( oneColumn ->Size() > MaximumSizeOfStemColumn)
2610 {
2611 continue;
2612 }
2613
2614 if ( oneColumn ->Size() == 1)
2615 {
2616 continue;
2617 }
2618
2619
2620 for ( itemi = 1; itemi <= oneColumn ->Size(); itemi++)
2621 {
2622
2623 Morphemei = oneColumn ->GetPiece(itemi).Display();
2624
2625 if ( i ==0)
2626 {
2627 Morphemei = Morphemei.right(Morphemei.length() -1);
2628 if ( Morphemei.length() ==0)
2629 {
2630 Morphemei = QString("NULL");
2631 continue;
2632 // NULL could not be any prefix or suffix of other morphemes
2633 }
2634
2635 }
2636
2637 if ( Morphemei == QString("NULL"))
2638 {
2639 continue;
2640 // NULL could not be any prefix or suffix of other morphemes
2641 }
2642
2643 if ( Morphemei.length() <= 1)
2644 {
2645 continue;
2646 // too short morpheme is suspicious for prefix and suffix of others
2647 }
2648
2649 ThePrefixOrSuffix = Morphemei; // Morphemei could be others' prefix or suffix
2650
2651 // Check through the other morpheme to see whether this morphemei could be prefix
2652 IsMePrefixOfOthers = false;
2653 for ( itemj= 1; itemj <= oneColumn ->Size(); itemj++)
2654 {
2655 Morphemej = oneColumn ->GetPiece(itemj).Display();
2656
2657 if ( i ==0)
2658 {
2659 Morphemej = Morphemej.right(Morphemej.length() -1);
2660 if ( Morphemej.length() ==0)
2661 {
2662 Morphemej = QString("NULL");
2663 }
2664
2665 continue;
2666 }
2667
2668
2669 if ( Morphemej.length() <= Morphemei.length())
2670 {
2671 continue;
2672 }
2673
2674 if ( Morphemej.left(Morphemei.length()) != Morphemei)
2675 {
2676 continue;
2677 }
2678
2679 IsMePrefixOfOthers = true;
2680 break;
2681 }
2682
2683 // If found morphemei is prefix of at least one other morpheme
2684 if ( IsMePrefixOfOthers)
2685 {
2686 CParse OriginalColumn, NewColumnOne, NewColumnTwo;
2687 CParse* ReplaceFirstColumn;
2688 int Originali;
2689 QString oneOriginalMoprheme, oneNewMorpheme;
2690 float OlderRobustNess, NewRobustNess;
2691 float RobustNessDiff;
2692 int Replacei;
2693 CParse** TempColumns;
2694 CParse* ReplaceOneColumn;
2695 QString Swappiece;
2696 CParse SwapParse;
2697
2698 // Try to get the NewColumnOne and NewColumnTwo
2699 for ( Originali = 1; Originali<= oneColumn ->Size(); Originali++)
2700 {
2701 oneOriginalMoprheme = oneColumn ->GetPiece(Originali).Display();
2702
2703 if ( i ==0)
2704 {
2705 oneOriginalMoprheme = oneOriginalMoprheme.right(oneOriginalMoprheme.length() -1);
2706
2707 if ( oneOriginalMoprheme.length() ==0)
2708 {
2709 oneOriginalMoprheme = QString("NULL");
2710 }
2711 }
2712
2713
2714 TempCSS = CStringSurrogate(oneOriginalMoprheme);
2715 OriginalColumn.Append(TempCSS);
2716
2717 if ( oneOriginalMoprheme == QString("NULL"))
2718 {
2719
2720 NewColumnOne.Append(TempCSS);
2721 continue;
2722 }
2723
2724 if ( oneOriginalMoprheme.length() <= ThePrefixOrSuffix.length())
2725 {
2726 NewColumnOne.Append(TempCSS);
2727 continue;
2728 }
2729
2730 if ( oneOriginalMoprheme.left(ThePrefixOrSuffix.length()) != ThePrefixOrSuffix)
2731 {
2732 NewColumnOne.Append(TempCSS);
2733 continue;
2734 }
2735
2736 if (oneOriginalMoprheme.left(ThePrefixOrSuffix.length()) == ThePrefixOrSuffix)
2737 {
2738 oneNewMorpheme = oneOriginalMoprheme.right(oneOriginalMoprheme.length() - ThePrefixOrSuffix.length());
2739
2740 TempCSS1 = CStringSurrogate(oneNewMorpheme);
2741
2742 NewColumnTwo.Append(TempCSS1);
2743 }
2744 }
2745
2746
2747 NewColumnTwo.Append(NULLCSS);
2748
2749 // Get Original Robustness
2750 OlderRobustNess = GetRobustNessWithParadigmaticGraph(OriginalColumn, GlobalNodeStickNess2, TotalGlobalStickNess2, GlobalStickNess2, TotalWord2);
2751
2752 // Calculate the alternative robustness to do
2753 NewRobustNess = GetRobustNessWithParadigmaticGraph(NewColumnOne, GlobalNodeStickNess2, TotalGlobalStickNess2, GlobalStickNess2, TotalWord2);
2754 NewRobustNess += GetRobustNessWithParadigmaticGraph(NewColumnTwo, GlobalNodeStickNess2, TotalGlobalStickNess2, GlobalStickNess2, TotalWord2);
2755
2756 // If Diff satisfy the condition, we make the modification, then break;
2757 RobustNessDiff = NewRobustNess - OlderRobustNess;
2758
2759 if (1) // RobustNessDiff < 0)
2760 {
2761
2762 // Change the template by replacing the two columns for the original one
2763 TempColumns = new CParse*[ m_NumberOfColumns + 1];
2764
2765 for ( Replacei = m_NumberOfColumns -1; Replacei >=0; Replacei--)
2766 {
2767 ReplaceOneColumn = m_Columns[Replacei];
2768
2769 if ( Replacei > i)
2770 {
2771 TempColumns[Replacei+1] = ReplaceOneColumn;
2772 }
2773
2774 if ( Replacei == i)
2775 {
2776 TempColumns[Replacei+1] = new CParse(NewColumnTwo);
2777 TempColumns[Replacei] = new CParse(NewColumnOne);
2778 delete m_Columns[Replacei];
2779 }
2780
2781 if ( Replacei < i)
2782 {
2783 TempColumns[Replacei] = ReplaceOneColumn;
2784 }
2785
2786 }
2787
2788 delete m_Columns;
2789 m_Columns = TempColumns;
2790 m_NumberOfColumns++;
2791
2792
2793
2794 // If necessary, Put back the "#" back for the first column
2795 if ( i ==0)
2796 {
2797
2798 ReplaceFirstColumn = m_Columns[0];
2799 for ( Replacei = 1; Replacei <= ReplaceFirstColumn ->Size(); Replacei++)
2800 {
2801 Swappiece = ReplaceFirstColumn ->GetPiece(Replacei).Display();
2802
2803 if ( Swappiece == QString("NULL"))
2804 {
2805 Swappiece = QString("#");
2806 }
2807 else
2808 {
2809 Swappiece = QString("#") + Swappiece;
2810 }
2811
2812 TempCSS = CStringSurrogate(Swappiece);
2813 SwapParse.Append(TempCSS);
2814 }
2815
2816 (*ReplaceFirstColumn)= SwapParse ;
2817 }
2818
2819 FoundAndModified = true;
2820 HasEverChanged = true;
2821 break;
2822 }
2823
2824 }
2825
2826
2827 // Check through the other morpheme to see whether this morphemei could be suffix
2828 IsMeSuffixOfOthers = false;
2829 for ( itemj= 1; itemj <= oneColumn ->Size(); itemj++)
2830 {
2831 Morphemej = oneColumn ->GetPiece(itemj).Display();
2832
2833 if ( i ==0)
2834 {
2835 Morphemej = Morphemej.right(Morphemej.length() -1);
2836 if ( Morphemej.length() ==0)
2837 {
2838 Morphemej = QString("NULL");
2839 }
2840
2841 continue;
2842 }
2843
2844
2845 if ( Morphemej.length() <= Morphemei.length())
2846 {
2847 continue;
2848 }
2849
2850 if ( Morphemej.right(Morphemei.length()) != Morphemei)
2851 {
2852 continue;
2853 }
2854
2855 IsMeSuffixOfOthers = true;
2856 break;
2857 }
2858
2859 // If found morphemei is prefix of at least one other morpheme
2860 if ( IsMeSuffixOfOthers)
2861 {
2862 CParse OriginalColumn, NewColumnOne, NewColumnTwo;
2863 CParse* ReplaceFirstColumn;
2864 int Originali;
2865 QString oneOriginalMoprheme, oneNewMorpheme;
2866 float OlderRobustNess, NewRobustNess;
2867 float RobustNessDiff;
2868 int Replacei;
2869 CParse** TempColumns;
2870 CParse* ReplaceOneColumn;
2871 QString Swappiece;
2872 CParse SwapParse;
2873
2874 // Try to get the NewColumnOne and NewColumnTwo
2875 for ( Originali = 1; Originali<= oneColumn ->Size(); Originali++)
2876 {
2877 oneOriginalMoprheme = oneColumn ->GetPiece(Originali).Display();
2878
2879 if ( i ==0)
2880 {
2881 oneOriginalMoprheme = oneOriginalMoprheme.right(oneOriginalMoprheme.length() -1);
2882
2883 if ( oneOriginalMoprheme.length() ==0)
2884 {
2885 oneOriginalMoprheme = QString("NULL");
2886 }
2887 }
2888
2889 TempCSS = CStringSurrogate(oneOriginalMoprheme);
2890 OriginalColumn.Append(TempCSS);
2891
2892 if ( oneOriginalMoprheme == QString("NULL"))
2893 {
2894 NewColumnTwo.Append(TempCSS);
2895 continue;
2896 }
2897
2898 if ( oneOriginalMoprheme.length() <= ThePrefixOrSuffix.length())
2899 {
2900 NewColumnTwo.Append(TempCSS);
2901 continue;
2902 }
2903
2904 if ( oneOriginalMoprheme.right(ThePrefixOrSuffix.length()) != ThePrefixOrSuffix)
2905 {
2906 NewColumnTwo.Append(TempCSS);
2907 continue;
2908 }
2909
2910 if (oneOriginalMoprheme.right(ThePrefixOrSuffix.length()) == ThePrefixOrSuffix)
2911 {
2912 oneNewMorpheme = oneOriginalMoprheme.left(oneOriginalMoprheme.length() - ThePrefixOrSuffix.length());
2913
2914 TempCSS1 = CStringSurrogate(oneNewMorpheme);
2915 NewColumnOne.Append(TempCSS1);
2916 }
2917 }
2918
2919 NewColumnOne.Append(NULLCSS);
2920
2921 // Get Original Robustness
2922 OlderRobustNess = GetRobustNessWithParadigmaticGraph(OriginalColumn, GlobalNodeStickNess2, TotalGlobalStickNess2, GlobalStickNess2, TotalWord2);
2923
2924 // Calculate the alternative robustness
2925 NewRobustNess = GetRobustNessWithParadigmaticGraph(NewColumnOne, GlobalNodeStickNess2, TotalGlobalStickNess2, GlobalStickNess2, TotalWord2);
2926 NewRobustNess += GetRobustNessWithParadigmaticGraph(NewColumnTwo, GlobalNodeStickNess2, TotalGlobalStickNess2, GlobalStickNess2, TotalWord2);
2927
2928 // If Diff satisfy the condition, we make the modification, then break;
2929 RobustNessDiff = NewRobustNess - OlderRobustNess;
2930
2931 if (1) //RobustNessDiff < 0)
2932 {
2933
2934 // Change the template by replacing the two columns for the original one
2935 TempColumns = new CParse*[ m_NumberOfColumns + 1];
2936
2937 for ( Replacei = m_NumberOfColumns -1; Replacei >=0; Replacei--)
2938 {
2939 ReplaceOneColumn = m_Columns[Replacei];
2940
2941 if ( Replacei > i)
2942 {
2943 TempColumns[Replacei+1] = ReplaceOneColumn;
2944 }
2945
2946 if ( Replacei == i)
2947 {
2948 TempColumns[Replacei+1] = new CParse(NewColumnTwo);
2949 TempColumns[Replacei] = new CParse(NewColumnOne);
2950 delete m_Columns[Replacei];
2951 }
2952
2953 if ( Replacei < i)
2954 {
2955 TempColumns[Replacei] = ReplaceOneColumn;
2956 }
2957
2958 }
2959
2960 delete m_Columns;
2961 m_Columns = TempColumns;
2962 m_NumberOfColumns++;
2963
2964
2965
2966 // if necessary, Put back the "#" back for the first column
2967
2968 if ( i == 0)
2969 {
2970
2971 ReplaceFirstColumn = m_Columns[0];
2972 for ( Replacei = 1; Replacei <= ReplaceFirstColumn ->Size(); Replacei++)
2973 {
2974 Swappiece = ReplaceFirstColumn ->GetPiece(Replacei).Display();
2975
2976 if ( Swappiece == QString("NULL"))
2977 {
2978 Swappiece = QString("#");
2979 }
2980 else
2981 {
2982 Swappiece = QString("#") + Swappiece;
2983 }
2984
2985 TempCSS = CStringSurrogate(Swappiece);
2986 SwapParse.Append(TempCSS);
2987 }
2988
2989 (*ReplaceFirstColumn)= SwapParse ;
2990 }
2991
2992 FoundAndModified = true;
2993 HasEverChanged = true;
2994 break;
2995 }
2996 }
2997
2998
2999 }
3000
3001
3002 if ( FoundAndModified) break;
3003
3004 }
3005
3006
3007 }// According to FoundAndModified, while loop;
3008
3009
3010 // If this template is modified, output to logf
3011 if ( HasEverChanged)
3012 {
3013 StringToParse OriginalWords, NewWords;
3014 StringToParse::iterator StringToParseIt;
3015 StringToInt NewCreatedWords;
3016 StringToInt::iterator StringToIntIt;
3017 QString TempWordString;
3018 CParse* dummyParse;
3019 QString DisplayOfOneColumn;
3020
3021
3022 QFile file( "PrefixOrSuffixMorphemeAdjust.txt" );
3023
3024 if ( !file.open( QIODevice::WriteOnly | QIODevice::Append) )
3025 {
3026 QMessageBox::information(NULL, "Error", "Can't Open the file!", "OK");
3027 return;
3028 }
3029
3030 Q3TextStream outf( &file );
3031
3032
3033
3034
3035 BackUpTemplate ->GetWordsAndParses(OriginalWords);
3036
3037
3038 for ( StringToParseIt = OriginalWords.begin(); StringToParseIt != OriginalWords.end(); StringToParseIt++)
3039 {
3040 dummyParse = StringToParseIt.data();
3041 delete dummyParse;
3042 }
3043
3044 this ->GetWordsAndParses(NewWords);
3045
3046 for ( StringToParseIt = NewWords.begin(); StringToParseIt != NewWords.end(); StringToParseIt++)
3047 {
3048 dummyParse = StringToParseIt.data();
3049 TempWordString = StringToParseIt.key();
3050
3051 delete dummyParse;
3052
3053 if ( !OriginalWords.contains(TempWordString))
3054 {
3055 NewCreatedWords.insert(TempWordString, NULL);
3056 }
3057
3058 }
3059
3060
3061
3062 outf << "***********"<<Loopi<<"************" <<endl;
3063 outf << " Original Template:" << endl;
3064 outf << " ";
3065
3066
3067 for ( i = 0; i < BackUpTemplate ->m_NumberOfColumns; i++)
3068 {
3069 oneColumn = BackUpTemplate ->m_Columns[i];
3070
3071 DisplayOfOneColumn = QString("{ ");
3072 for (int j = 1; j <= oneColumn ->Size();j++)
3073 {
3074 DisplayOfOneColumn += oneColumn ->GetPiece(j).Display();
3075 if ( j != oneColumn ->Size())
3076 {
3077 DisplayOfOneColumn += QString(" , ");
3078 }
3079 }
3080
3081 DisplayOfOneColumn += QString(" }");
3082 outf << DisplayOfOneColumn;
3083 if ( i != m_NumberOfColumns -1)
3084 {
3085 outf << " --> ";
3086 }
3087 }
3088
3089
3090 outf << endl;
3091 outf << endl;
3092
3093 outf << " New Template:" << endl;
3094 outf << " ";
3095
3096 for ( i = 0; i < m_NumberOfColumns; i++)
3097 {
3098 oneColumn = m_Columns[i];
3099
3100 DisplayOfOneColumn = QString("{ ");
3101 for (int j = 1; j <= oneColumn ->Size();j++)
3102 {
3103 DisplayOfOneColumn += oneColumn ->GetPiece(j).Display();
3104 if ( j != oneColumn ->Size())
3105 {
3106 DisplayOfOneColumn += QString(" , ");
3107 }
3108 }
3109
3110 DisplayOfOneColumn += QString(" }");
3111 outf << DisplayOfOneColumn;
3112 if ( i != m_NumberOfColumns -1)
3113 {
3114 outf << " --> ";
3115 }
3116 }
3117
3118 outf << endl;
3119 outf << endl;
3120
3121 outf << " New Created Words:" << endl;
3122 outf << " { ";
3123
3124 for ( StringToIntIt = NewCreatedWords.begin(); StringToIntIt != NewCreatedWords.end(); StringToIntIt++)
3125 {
3126 TempWordString = StringToIntIt.key();
3127 outf << TempWordString << " , ";
3128 }
3129
3130 outf << " }"<< endl;
3131 outf << endl;
3132 outf << endl;
3133 outf << endl;
3134
3135 file.close();
3136
3137 }
3138
3139
3140 delete BackUpTemplate;
3141
3142 }
3143
3144
3145
3146 float CTemplate::GetRobustNessWithParadigmaticGraph(CParse& oneColumn, StringToFloat& MorphemeAndItsComplexity, StringToStringToFloat& GlobalStickNess, float TotalGlobalStickNess, float TotalWords)
3147 {
3148
3149
3150 int i, j;
3151 QString oneMorpheme, anotherMorpheme;
3152 QString HostMorpheme, SlaveMorpheme;
3153 float NodeRobustNess;
3154 float EdgeRobustNess;
3155 float oneFloat;
3156 StringToFloat* oneCollection;
3157
3158
3159 NodeRobustNess =0.0;
3160 EdgeRobustNess = 0.0;
3161
3162
3163 for ( i=1; i<= oneColumn.Size(); i++)
3164 {
3165 oneMorpheme = oneColumn.GetPiece(i).Display();
3166
3167 if (MorphemeAndItsComplexity.contains(oneMorpheme))
3168 {
3169 oneFloat = MorphemeAndItsComplexity[oneMorpheme];
3170 NodeRobustNess += oneFloat;
3171 }
3172 else
3173 {
3174 NodeRobustNess += (-base2log(1.0/TotalWords)); // Big Penalty
3175
3176 }
3177
3178 for ( j =i+1; j<= oneColumn.Size(); j++)
3179 {
3180 anotherMorpheme = oneColumn.GetPiece(j).Display();
3181
3182 if ( oneMorpheme > anotherMorpheme )
3183 {
3184 if ( GlobalStickNess.contains(oneMorpheme))
3185 {
3186 oneCollection = GlobalStickNess[oneMorpheme];
3187
3188 if ( oneCollection ->contains(anotherMorpheme))
3189 {
3190 oneFloat = (*oneCollection)[anotherMorpheme];
3191 EdgeRobustNess += oneFloat;
3192 }
3193 else
3194 {
3195 EdgeRobustNess += -base2log(1.0/TotalGlobalStickNess); // Big Penalty
3196
3197 }
3198
3199 }
3200 else
3201 {
3202 EdgeRobustNess += -base2log(1.0/TotalGlobalStickNess); // Big Penalty
3203 }
3204 }
3205 else
3206 {
3207 if ( GlobalStickNess.contains(anotherMorpheme))
3208 {
3209 oneCollection = GlobalStickNess[anotherMorpheme];
3210
3211 if ( oneCollection ->contains(oneMorpheme))
3212 {
3213 oneFloat = (*oneCollection)[oneMorpheme];
3214 EdgeRobustNess += oneFloat;
3215 }
3216 else
3217 {
3218 EdgeRobustNess += -base2log(1.0/TotalGlobalStickNess); // Big Penalty
3219
3220 }
3221
3222 }
3223 else
3224 {
3225 EdgeRobustNess += -base2log(1.0/TotalGlobalStickNess); // Big Penalty
3226 }
3227 }
3228
3229 }
3230
3231
3232 }
3233
3234 return (NodeRobustNess + EdgeRobustNess);
3235 }
3236
3237
3238
3239
3240
3241
3242
3243
3244 /*
3245 // Answer the Q: does this template describe prefixes, or suffixes?
3246 eAffixationType CTemplate::DetermineAffixationSide()
3247 {
3248 float threshold = 2;
3249 int MinimumNumberOfStems = 3;
3250 int ColumnNumber = 0;
3251 int MinimumStemLength = 4;
3252
3253 if ( m_NumberOfColumns != 2 ) return TYPE_Unknown;
3254
3255 //#############################################################/
3256 // initial templates have only 1 column with more than 1 entry; deal with them first
3257 int NumberOfColumnsWithMoreThan1Entry = 0;
3258 for (int i = 0; i < m_NumberOfColumns; i++)
3259 { if ( m_Columns[i]->Size() > 1 )
3260 NumberOfColumnsWithMoreThan1Entry ++;
3261 ColumnNumber = i;
3262 }
3263 //###########################################################//
3264 if ( NumberOfColumnsWithMoreThan1Entry == 1 )
3265 {
3266 int OtherColumn = 2 - ColumnNumber;
3267 }
3268 //###########################################################//
3269 else
3270 {
3271
3272
3273 // We consider regular templates, with more than one column containing more than 1 entry
3274 float N1 = (float) m_Columns[0]->GetLength();
3275 float N2 = (float) m_Columns[1]->GetLength();
3276
3277
3278 if ( N1/N2 > threshold && m_Columns[0]->Size() >= MinimumNumberOfStems )
3279 {
3280 return TYPE_Suffix;
3281 }
3282 if ( N2/N1 > threshold && m_Columns[1]->Size() >= MinimumNumberOfStems)
3283 {
3284 return TYPE_Prefix;
3285 }
3286 return TYPE_Unknown;
3287
3288 }
3289 //#########################################################//
3290
3291 return TYPE_Unknown;
3292 }
3293
3294 void CTemplate::AddToCollections ( eAffixationType ThisType, CStemCollection* Stems, CAffixCollection* Affixes, CSignatureCollection* Signatures)
3295 {
3296
3297 CStem* pStem;
3298 CSignature* pSig,
3299 *qSig;
3300 CParse Sig1, Sig2;
3301 CStringSurrogate ssPiece;
3302 CString WordBoundary("#"),
3303 Null ("NULL");
3304 // Right now, there is a spurious '#' at the beginning of words, so we have to
3305 // remove this from each prefix and prefix signature. The # should be removed,
3306 // and this code simplified, so that prefixes are just like suffixes.
3307 //
3308 if (ThisType == TYPE_Prefix)
3309 {
3310 for (int i = 1; i <= m_Columns[0]->Size(); i++)
3311 {
3312 Sig1 = m_Columns[0]->GetAt_SS(i);
3313 for (int i=1; i <= Sig1.Size(); i++)
3314 {
3315 ssPiece = Sig1.GetAt_SS(i);
3316 if ( ssPiece == WordBoundary )
3317 {
3318 Sig2.AppendInAlphabeticalOrder (Null );
3319 ssPiece = Null;
3320 }
3321 else if ( ssPiece[0] == '#' )
3322 {
3323 ssPiece = ssPiece.Mid(1);
3324 Sig2.AppendInAlphabeticalOrder ( ssPiece );
3325 }
3326 *Affixes << ssPiece;
3327 }
3328 qSig = *Signatures << &Sig2;
3329 qSig->SetAffixLocation ( WORD_INITIAL ); //this is probably no longer necessary; todo
3330 }
3331 for (i = 1; i <= m_Columns[1]->Size(); i++)
3332 {
3333 pStem = *Stems << m_Columns[1]->GetAt_SS(i);
3334 qSig->GetStemPtrList()->AddTail(pStem);
3335 pStem->SetSuffixSignature (qSig);
3336 }
3337
3338 return;
3339 }
3340 else if ( ThisType == TYPE_Suffix)
3341 {
3342 pSig = *Signatures << m_Columns[1];
3343
3344 for (int i = 1; i <= m_Columns[1]->Size(); i++)
3345 {
3346 *Affixes << m_Columns[1]->GetAt_SS(i);
3347 }
3348 for (i = 1; i <= m_Columns[0]->Size(); i++)
3349 {
3350 pStem = *Stems << m_Columns[0]->GetAt_SS(i);
3351 pSig->GetStemPtrList()->AddTail(pStem);
3352 pStem->SetSuffixSignature (pSig);
3353 }
3354
3355
3356 return;
3357 }
3358 return;
3359
3360
3361
3362
3363 }
3364
3365 */
3366
3367
3368 //////////////////////////////////////////////////////////////////
3369 ////// CStateListViewItem Implementation Section
3370
3371 CTemplateListViewItem::CTemplateListViewItem(Q3ListView *parent,
3372 CTemplate* templ, QString label)
3373 : Q3ListViewItem(parent, label),
3374 m_Template(templ),
3375 m_MaxMumShownMorphemes(4) { }
3376
3377 CTemplateListViewItem::CTemplateListViewItem(Q3ListViewItem* parent,
3378 CTemplate* templ, QString label)
3379 : Q3ListViewItem(parent, label),
3380 m_Template(templ),
3381 m_MaxMumShownMorphemes(4) { }
3382
3383 QString CTemplateListViewItem::key( int column, bool ascending ) const
3384 {
3385 if(m_Template)
3386 {
3387 QString QSstring = "";
3388 int NumberOfMorphemes;
3389
3390
3391 switch( column )
3392 {
3393 case 0:
3394 return QString("%1").arg(m_Template ->m_TemplateNumber,10);
3395 case 1:
3396 return QString("%1").arg(m_Template ->m_NumberOfColumns,10);
3397 case 2:
3398 return QString("%1").arg(m_Template ->m_NewSortComplexity, 10);
3399 case 3:
3400
3401 if (m_Template-> m_Columns[0] )
3402 {
3403 NumberOfMorphemes = m_Template->m_Columns[0]->Size() ;
3404 }
3405 else
3406 {
3407 NumberOfMorphemes = 0;
3408 }
3409 return QString("%1").arg(NumberOfMorphemes,10);
3410
3411 case 4:
3412
3413 if (m_Template-> m_Columns[1] )
3414 {
3415 NumberOfMorphemes = m_Template->m_Columns[1]->Size() ;
3416 }
3417 else
3418 {
3419 NumberOfMorphemes = 0;
3420 }
3421 return QString("%1").arg(NumberOfMorphemes,10);
3422
3423 case 5:
3424
3425 if (m_Template->m_NumberOfColumns >= 3 )
3426 {
3427 NumberOfMorphemes = m_Template->m_Columns[2]->Size() ;
3428 }
3429 else
3430 {
3431 NumberOfMorphemes = 0;
3432 }
3433 return QString("%1").arg(NumberOfMorphemes,10);
3434
3435 case 6:
3436
3437 if (m_Template->m_NumberOfColumns >= 4 )
3438 {
3439 NumberOfMorphemes = m_Template->m_Columns[3]->Size() ;
3440 }
3441 else
3442 {
3443 NumberOfMorphemes = 0;
3444 }
3445 return QString("%1").arg(NumberOfMorphemes,10);
3446
3447 default:
3448 return Q3ListViewItem::text( column );
3449 }
3450 }
3451 else return Q3ListViewItem::key( column, ascending );
3452
3453
3454 }
3455
3456 QString CTemplateListViewItem::text( int column ) const
3457 {
3458
3459 if(m_Template)
3460 {
3461 QString QSstring = "";
3462 int i;
3463 int NumberOfMorphemes;
3464 int NumberOfDisplayedMorphemes;
3465 bool ShowPlus;
3466 // int PrintedMorphemes=0;
3467 QString QSOneMorpheme, QSReversedOneMorpheme;
3468 CStringSurrogate SSOneMorpheme;
3469 // const char* CCDebugString1, *CCDebugString2;
3470
3471
3472 switch( column )
3473 {
3474 case 0:
3475 return QString("%1").arg( m_Template->m_TemplateNumber);
3476 case 1:
3477 return QString("%1").arg( m_Template ->m_NumberOfColumns);
3478 case 2:
3479 return QString("%1").arg( m_Template ->m_NewSortComplexity);
3480 case 3:
3481
3482
3483 if ( m_Template->m_NumberOfColumns < 1)
3484 {
3485 return QString("");
3486 }
3487
3488 NumberOfMorphemes = m_Template->m_Columns[0] ->Size() ;
3489
3490 NumberOfDisplayedMorphemes = m_MaxMumShownMorphemes;
3491 ShowPlus = true;
3492 if(NumberOfMorphemes <= m_MaxMumShownMorphemes)
3493 {
3494 NumberOfDisplayedMorphemes = NumberOfMorphemes;
3495 ShowPlus = false;
3496 }
3497
3498 for( i = 1; i <= NumberOfDisplayedMorphemes ; i++ )
3499 {
3500 QSstring.append( m_Template->m_Columns[0] ->GetPiece(i).Display() + ", ");
3501 }
3502 QSstring = QSstring.left( QSstring.length() - 2 );
3503
3504 if(ShowPlus)
3505 {
3506 QSstring += " +";
3507 }
3508 return QSstring;
3509
3510 case 4:
3511
3512 if ( m_Template->m_NumberOfColumns < 2)
3513 {
3514 return QString("");
3515 }
3516
3517 NumberOfMorphemes = m_Template->m_Columns[1] ->Size() ;
3518
3519 NumberOfDisplayedMorphemes = m_MaxMumShownMorphemes;
3520 ShowPlus = true;
3521 if(NumberOfMorphemes <= m_MaxMumShownMorphemes)
3522 {
3523 NumberOfDisplayedMorphemes = NumberOfMorphemes;
3524 ShowPlus = false;
3525 }
3526
3527 for( i = 1; i <= NumberOfDisplayedMorphemes ; i++ )
3528 {
3529 QSstring.append( m_Template->m_Columns[1] ->GetPiece(i).Display() + ", ");
3530 }
3531 QSstring = QSstring.left( QSstring.length() - 2 );
3532
3533 if(ShowPlus)
3534 {
3535 QSstring += " +";
3536 }
3537 return QSstring;
3538
3539 case 5:
3540
3541 if ( m_Template->m_NumberOfColumns < 3)
3542 {
3543 return QString("");
3544 }
3545
3546 NumberOfMorphemes = m_Template->m_Columns[2] ->Size() ;
3547
3548 NumberOfDisplayedMorphemes = m_MaxMumShownMorphemes;
3549 ShowPlus = true;
3550 if(NumberOfMorphemes <= m_MaxMumShownMorphemes)
3551 {
3552 NumberOfDisplayedMorphemes = NumberOfMorphemes;
3553 ShowPlus = false;
3554 }
3555
3556 for( i = 1; i <= NumberOfDisplayedMorphemes ; i++ )
3557 {
3558 QSstring.append( m_Template->m_Columns[2] ->GetPiece(i).Display() + ", ");
3559 }
3560 QSstring = QSstring.left( QSstring.length() - 2 );
3561
3562 if(ShowPlus)
3563 {
3564 QSstring += " +";
3565 }
3566 return QSstring;
3567
3568 case 6:
3569
3570 if ( m_Template->m_NumberOfColumns < 4)
3571 {
3572 return QString("");
3573 }
3574
3575 NumberOfMorphemes = m_Template->m_Columns[3] ->Size() ;
3576
3577 NumberOfDisplayedMorphemes = m_MaxMumShownMorphemes;
3578 ShowPlus = true;
3579 if(NumberOfMorphemes <= m_MaxMumShownMorphemes)
3580 {
3581 NumberOfDisplayedMorphemes = NumberOfMorphemes;
3582 ShowPlus = false;
3583 }
3584
3585 for( i = 1; i <= NumberOfDisplayedMorphemes ; i++ )
3586 {
3587 QSstring.append( m_Template->m_Columns[3] ->GetPiece(i).Display() + ", ");
3588 }
3589 QSstring = QSstring.left( QSstring.length() - 2 );
3590
3591 if(ShowPlus)
3592 {
3593 QSstring += " +";
3594 }
3595 return QSstring;
3596
3597
3598 default:
3599 return Q3ListViewItem::text( column );
3600 }
3601 }
3602 else return Q3ListViewItem::text( column );
3603
3604 }
3605
3606
automatic morphological analysis of a text