Returns chunks of a string seperated by a specified delimiter string containing any number of characters.

A token is basically a chunk, or substring, of a string. Handles multiple character delimiters. nextToken$() is an iterative function. So everytime you call it, it'll give the next token from the string.

                                    ` This code was downloaded from The Game Creators
                                    ` It is reproduced here with full permission
                                    ` http://www.thegamecreators.com
                                    
                                    Global delimiter$ as string
Global string$ as string
Global token$ as string
Global stringLength as integer
Global startingPosition as integer
Global delimiterLength as integer
delimiter$= ", "
string$ = "906,Sat,28,Aug,2004,06,07,10,20,37,44,36, 1263778, 4, Pearl , 7"
token$ = ""
stringLength = len(string$)
startingPosition = 1
delimiterLength = len(delimiter$)



while hasMoreTokens() = 1
  print nextToken$()
endwhile

repeat : until spacekey()




REM Returns the next token from the string
function nextToken$()
  flag = 0

  if delimiter$ = ""
      startingPosition = stringLength
      token$ = string$
  endif

  for i = startingPosition to stringLength-1
      for x = 1 to delimiterLength
        if mid$(delimiter$,x) = mid$(string$,i)
            token$ = subString(string$, startingPosition-1, i-1)
            startingPosition = i+1
            flag = 1
            sp = startingPosition
            repeat
               hit = 0
               for x = 1 to delimiterLength
                  if mid$(delimiter$,x) = mid$(string$,sp) then hit = 1
               next x
               if hit = 1 then inc sp, 1
            until hit = 0
            startingPosition = sp
            exit
        endif
        if (i = stringLength-1)
            token$ = subString(string$, startingPosition-1, i+1)
            startingPosition = i+1
            flag = 1
        endif
      next x

      if flag = 1 then exit
  next i
endfunction token$



REM Returns true(1) if the string has more tokens left
function hasMoreTokens()
  if startingPosition <= stringLength-1 then exitfunction 1
endfunction 0



REM Returns a chunk of a whole string
REM starp - is the starting position of the substring that
REM        you want to be returned. The character at that
REM        position is not included in the returned result.
REM endp - is the position of the last character in the string
REM        you want to be returned.
function subString(s$ as string, startp as integer, endp as integer)
  l = len(s$)
  s$ = right$(s$,l-startp)
  s$ = left$(s$, endp-startp)
endfunction s$