Changesets can be listed by changeset number.
The Git repository is here.
- Revision:
- 15
- Log:
Attempt to update Typo to a Typo SVN HEAD release from around the
time the prototype installation was set up on the RISC OS Open Limited
web site. Timestamps place this at 04-Jul so a revision from 05-Jul or
earlier was pulled and copied over the 2.6.0 tarball stable code.
- Author:
- adh
- Date:
- Sat Jul 22 23:27:35 +0100 2006
- Size:
- 11329 Bytes
1 | <?xml version="1.0" encoding="iso-8859-1"?> |
2 | <!DOCTYPE html |
3 | PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" |
4 | "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"> |
5 | <html> |
6 | <head> |
7 | <title>Class: Syntax::Tokenizer</title> |
8 | <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" /> |
9 | <link rel="stylesheet" href="../.././rdoc-style.css" type="text/css" media="screen" /> |
10 | |
11 | <script language="JavaScript" type="text/javascript"> |
12 | // <![CDATA[ |
13 | |
14 | function toggleSource( id ) |
15 | { |
16 | var elem |
17 | var link |
18 | |
19 | if( document.getElementById ) |
20 | { |
21 | elem = document.getElementById( id ) |
22 | link = document.getElementById( "l_" + id ) |
23 | } |
24 | else if ( document.all ) |
25 | { |
26 | elem = eval( "document.all." + id ) |
27 | link = eval( "document.all.l_" + id ) |
28 | } |
29 | else |
30 | return false; |
31 | |
32 | if( elem.style.display == "block" ) |
33 | { |
34 | elem.style.display = "none" |
35 | link.innerHTML = "show source" |
36 | } |
37 | else |
38 | { |
39 | elem.style.display = "block" |
40 | link.innerHTML = "hide source" |
41 | } |
42 | } |
43 | |
44 | function openCode( url ) |
45 | { |
46 | window.open( url, "SOURCE_CODE", "width=400,height=400,scrollbars=yes" ) |
47 | } |
48 | // ]]> |
49 | </script> |
50 | </head> |
51 | |
52 | <body> |
53 | <table width="100%" border='0' cellpadding='0' cellspacing='0' class='banner'><tr> |
54 | <td class="file-title"><span class="file-title-prefix">Class</span><br />Syntax::Tokenizer</td> |
55 | <td align="right"> |
56 | <table cellspacing=0 cellpadding=2> |
57 | <tr valign="top"> |
58 | <td>In:</td> |
59 | <td> |
60 | <a href="../../files/lib/syntax/common_rb.html">lib/syntax/common.rb</a> |
61 | </td> |
62 | </tr> |
63 | <tr> |
64 | <td>Parent:</td> |
65 | <td> |
66 | Object |
67 | </td> |
68 | </tr> |
69 | </table> |
70 | </td> |
71 | </tr> |
72 | </table> |
73 | <!-- banner header --> |
74 | |
75 | <div id="bodyContent"> |
76 | <div id="content"> |
77 | |
78 | <div class="description"><p> |
79 | The base class of all tokenizers. It sets up the scanner and manages the |
80 | looping until all tokens have been extracted. It also provides convenience |
81 | methods to make sure adjacent tokens of identical groups are returned as a |
82 | single token. |
83 | </p> |
84 | </div> |
85 | |
86 | |
87 | |
88 | <div class="sectiontitle">Methods</div> |
89 | <ul> |
90 | <li><a href="#M000012">finish</a></li> |
91 | <li><a href="#M000017">option</a></li> |
92 | <li><a href="#M000016">set</a></li> |
93 | <li><a href="#M000011">setup</a></li> |
94 | <li><a href="#M000010">start</a></li> |
95 | <li><a href="#M000014">step</a></li> |
96 | <li><a href="#M000013">teardown</a></li> |
97 | <li><a href="#M000015">tokenize</a></li> |
98 | </ul> |
99 | |
100 | |
101 | |
102 | |
103 | <div class="sectiontitle">Constants</div> |
104 | <table border='0' cellpadding='5'> |
105 | <tr valign='top'> |
106 | <td class="attr-name">EOL</td> |
107 | <td>=</td> |
108 | <td class="attr-value">/(?=\r\n?|\n|$)/</td> |
109 | </tr> |
110 | </table> |
111 | |
112 | <div class="sectiontitle">Attributes</div> |
113 | <table border='0' cellpadding='5'> |
114 | <tr valign='top'> |
115 | <td class='attr-rw'> |
116 | [R] |
117 | </td> |
118 | <td class='attr-name'>chunk</td> |
119 | <td class='attr-desc'> |
120 | The current chunk of text being accumulated |
121 | |
122 | </td> |
123 | </tr> |
124 | <tr valign='top'> |
125 | <td class='attr-rw'> |
126 | [R] |
127 | </td> |
128 | <td class='attr-name'>group</td> |
129 | <td class='attr-desc'> |
130 | The current group being processed by the tokenizer |
131 | |
132 | </td> |
133 | </tr> |
134 | </table> |
135 | |
136 | <div class="sectiontitle">Public Instance methods</div> |
137 | <div class="method"> |
138 | <div class="title"> |
139 | <a name="M000012"></a><b>finish</b>() |
140 | </div> |
141 | <div class="description"> |
142 | <p> |
143 | Finish tokenizing. This flushes the buffer, yielding any remaining text to |
144 | the client. |
145 | </p> |
146 | </div> |
147 | <div class="sourcecode"> |
148 | <p class="source-link">[ <a href="javascript:toggleSource('M000012_source')" id="l_M000012_source">show source</a> ]</p> |
149 | <div id="M000012_source" class="dyn-source"> |
150 | <pre> |
151 | <span class="ruby-comment cmt"># File lib/syntax/common.rb, line 57</span> |
152 | 57: <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">finish</span> |
153 | 58: <span class="ruby-identifier">start_group</span> <span class="ruby-keyword kw">nil</span> |
154 | 59: <span class="ruby-identifier">teardown</span> |
155 | 60: <span class="ruby-keyword kw">end</span> |
156 | </pre> |
157 | </div> |
158 | </div> |
159 | </div> |
160 | <div class="method"> |
161 | <div class="title"> |
162 | <a name="M000017"></a><b>option</b>(opt) |
163 | </div> |
164 | <div class="description"> |
165 | <p> |
166 | Get the value of the specified option. |
167 | </p> |
168 | </div> |
169 | <div class="sourcecode"> |
170 | <p class="source-link">[ <a href="javascript:toggleSource('M000017_source')" id="l_M000017_source">show source</a> ]</p> |
171 | <div id="M000017_source" class="dyn-source"> |
172 | <pre> |
173 | <span class="ruby-comment cmt"># File lib/syntax/common.rb, line 89</span> |
174 | 89: <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">option</span>(<span class="ruby-identifier">opt</span>) |
175 | 90: <span class="ruby-ivar">@options</span> <span class="ruby-operator">?</span> <span class="ruby-ivar">@options</span>[<span class="ruby-identifier">opt</span>] <span class="ruby-operator">:</span> <span class="ruby-keyword kw">nil</span> |
176 | 91: <span class="ruby-keyword kw">end</span> |
177 | </pre> |
178 | </div> |
179 | </div> |
180 | </div> |
181 | <div class="method"> |
182 | <div class="title"> |
183 | <a name="M000016"></a><b>set</b>( opts={} ) |
184 | </div> |
185 | <div class="description"> |
186 | <p> |
187 | Specify a set of tokenizer-specific options. Each tokenizer may (or may |
188 | not) publish any options, but if a tokenizer does those options may be used |
189 | to specify optional behavior. |
190 | </p> |
191 | </div> |
192 | <div class="sourcecode"> |
193 | <p class="source-link">[ <a href="javascript:toggleSource('M000016_source')" id="l_M000016_source">show source</a> ]</p> |
194 | <div id="M000016_source" class="dyn-source"> |
195 | <pre> |
196 | <span class="ruby-comment cmt"># File lib/syntax/common.rb, line 84</span> |
197 | 84: <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">set</span>( <span class="ruby-identifier">opts</span>={} ) |
198 | 85: ( <span class="ruby-ivar">@options</span> <span class="ruby-operator">||=</span> <span class="ruby-constant">Hash</span>.<span class="ruby-identifier">new</span> ).<span class="ruby-identifier">update</span> <span class="ruby-identifier">opts</span> |
199 | 86: <span class="ruby-keyword kw">end</span> |
200 | </pre> |
201 | </div> |
202 | </div> |
203 | </div> |
204 | <div class="method"> |
205 | <div class="title"> |
206 | <a name="M000011"></a><b>setup</b>() |
207 | </div> |
208 | <div class="description"> |
209 | <p> |
210 | Subclasses may override this method to provide implementation-specific |
211 | setup logic. |
212 | </p> |
213 | </div> |
214 | <div class="sourcecode"> |
215 | <p class="source-link">[ <a href="javascript:toggleSource('M000011_source')" id="l_M000011_source">show source</a> ]</p> |
216 | <div id="M000011_source" class="dyn-source"> |
217 | <pre> |
218 | <span class="ruby-comment cmt"># File lib/syntax/common.rb, line 52</span> |
219 | 52: <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">setup</span> |
220 | 53: <span class="ruby-keyword kw">end</span> |
221 | </pre> |
222 | </div> |
223 | </div> |
224 | </div> |
225 | <div class="method"> |
226 | <div class="title"> |
227 | <a name="M000010"></a><b>start</b>( text, &block ) |
228 | </div> |
229 | <div class="description"> |
230 | <p> |
231 | Start tokenizing. This sets up the state in preparation for tokenization, |
232 | such as creating a new scanner for the text and saving the callback block. |
233 | The block will be invoked for each token extracted. |
234 | </p> |
235 | </div> |
236 | <div class="sourcecode"> |
237 | <p class="source-link">[ <a href="javascript:toggleSource('M000010_source')" id="l_M000010_source">show source</a> ]</p> |
238 | <div id="M000010_source" class="dyn-source"> |
239 | <pre> |
240 | <span class="ruby-comment cmt"># File lib/syntax/common.rb, line 42</span> |
241 | 42: <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">start</span>( <span class="ruby-identifier">text</span>, <span class="ruby-operator">&</span><span class="ruby-identifier">block</span> ) |
242 | 43: <span class="ruby-ivar">@chunk</span> = <span class="ruby-value str">""</span> |
243 | 44: <span class="ruby-ivar">@group</span> = <span class="ruby-identifier">:normal</span> |
244 | 45: <span class="ruby-ivar">@callback</span> = <span class="ruby-identifier">block</span> |
245 | 46: <span class="ruby-ivar">@text</span> = <span class="ruby-constant">StringScanner</span>.<span class="ruby-identifier">new</span>( <span class="ruby-identifier">text</span> ) |
246 | 47: <span class="ruby-identifier">setup</span> |
247 | 48: <span class="ruby-keyword kw">end</span> |
248 | </pre> |
249 | </div> |
250 | </div> |
251 | </div> |
252 | <div class="method"> |
253 | <div class="title"> |
254 | <a name="M000014"></a><b>step</b>() |
255 | </div> |
256 | <div class="description"> |
257 | <p> |
258 | Subclasses must implement this method, which is called for each iteration |
259 | of the tokenization process. This method may extract multiple tokens. |
260 | </p> |
261 | </div> |
262 | <div class="sourcecode"> |
263 | <p class="source-link">[ <a href="javascript:toggleSource('M000014_source')" id="l_M000014_source">show source</a> ]</p> |
264 | <div id="M000014_source" class="dyn-source"> |
265 | <pre> |
266 | <span class="ruby-comment cmt"># File lib/syntax/common.rb, line 69</span> |
267 | 69: <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">step</span> |
268 | 70: <span class="ruby-identifier">raise</span> <span class="ruby-constant">NotImplementedError</span>, <span class="ruby-value str">"subclasses must implement #step"</span> |
269 | 71: <span class="ruby-keyword kw">end</span> |
270 | </pre> |
271 | </div> |
272 | </div> |
273 | </div> |
274 | <div class="method"> |
275 | <div class="title"> |
276 | <a name="M000013"></a><b>teardown</b>() |
277 | </div> |
278 | <div class="description"> |
279 | <p> |
280 | Subclasses may override this method to provide implementation-specific |
281 | teardown logic. |
282 | </p> |
283 | </div> |
284 | <div class="sourcecode"> |
285 | <p class="source-link">[ <a href="javascript:toggleSource('M000013_source')" id="l_M000013_source">show source</a> ]</p> |
286 | <div id="M000013_source" class="dyn-source"> |
287 | <pre> |
288 | <span class="ruby-comment cmt"># File lib/syntax/common.rb, line 64</span> |
289 | 64: <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">teardown</span> |
290 | 65: <span class="ruby-keyword kw">end</span> |
291 | </pre> |
292 | </div> |
293 | </div> |
294 | </div> |
295 | <div class="method"> |
296 | <div class="title"> |
297 | <a name="M000015"></a><b>tokenize</b>( text, &block ) |
298 | </div> |
299 | <div class="description"> |
300 | <p> |
301 | Begins tokenizing the given text, calling <a |
302 | href="Tokenizer.html#M000014">step</a> until the text has been exhausted. |
303 | </p> |
304 | </div> |
305 | <div class="sourcecode"> |
306 | <p class="source-link">[ <a href="javascript:toggleSource('M000015_source')" id="l_M000015_source">show source</a> ]</p> |
307 | <div id="M000015_source" class="dyn-source"> |
308 | <pre> |
309 | <span class="ruby-comment cmt"># File lib/syntax/common.rb, line 75</span> |
310 | 75: <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">tokenize</span>( <span class="ruby-identifier">text</span>, <span class="ruby-operator">&</span><span class="ruby-identifier">block</span> ) |
311 | 76: <span class="ruby-identifier">start</span> <span class="ruby-identifier">text</span>, <span class="ruby-operator">&</span><span class="ruby-identifier">block</span> |
312 | 77: <span class="ruby-identifier">step</span> <span class="ruby-keyword kw">until</span> <span class="ruby-ivar">@text</span>.<span class="ruby-identifier">eos?</span> |
313 | 78: <span class="ruby-identifier">finish</span> |
314 | 79: <span class="ruby-keyword kw">end</span> |
315 | </pre> |
316 | </div> |
317 | </div> |
318 | </div> |
319 | </div> |
320 | |
321 | </div> |
322 | |
323 | </body> |
324 | </html> |