How to check for balanced brackets WITHOUT a stack/regex?

Question

So basically before people start questioning why I'm not using a stack to save time over using counters and stuff. This is a homework problem working with space complexity, so ignoring time complexity, we are attempting to reduce space complexity.

To do so, I have to use counters to keep track of the brackets.

Possible bracket types: '(' ')' '[' ']'

I've tried some coding but I seem to be having a problem with one of the test strings, and I just can't pinpoint where the problem is happening.

Boolean isWF(String w) {
// maxLevel is found and any variables I'm using has been initialized

  for(int i = 0; i < maxLevel; i++) {
      x = w.charAt(i);
      currentLevel = i;

      if(x == '(' || x == '[') {
        holder = x; // Store bracket here to check match
        savedLevel++;
        counter++;
        currentLevel++;

        for(int j = i+1; j < w.length(); j++) {
          x = w.charAt(j);

          if(x == '(' || x == '[') {
            currentLevel++; 
            if(currentLevel == savedLevel) {
              holder = x;
              counter++;
            }
          }
          else if(x == ')' || x == ']') {
            if(currentLevel == savedLevel) {
              if((holder == '(' && x == ')') || (holder == '[' && x == ']')) {
                currentLevel--;
                counter--;
              }
              else
                return false;
            }
            else {
              currentLevel--;
            }
          }
        }
      }
      else if(x == ')' || x == ']') {
        counter--;
        if(counter < 0) {
          return false;
        }
      }
    }
    if(counter != 0) {
      return false;
    }

    return true;
  }
}

The strings I'm testing:

()[] - expected to be true, actual is true
([)] - expected to be false, actual is false
[([([()])])] - expected to be true, actual is true
([()([])()][()(())]()) - expected to be true, actual is false
([()([])())[()(())]()) - expected to be false, actual is false

Answer 1

Not a direct answer to where is the bug in your approach but the following approach seems to solve your input cases and is much simpler. Basically you go over the string checking if the next symbol is one you can accept e.g. you can't accept a ) right after a [ and you keep a count of the open/close of brackets. If they ever go negative it means you are missing something.

public static boolean isBalanced(String s) {
        int sOpen = 0;
        int rOpen = 0;
        for(int i = 0; i < s.length() - 1; ++i) {
            final char current = s.charAt(i);
            final char next = s.charAt(i + 1);
            if(!isValidSymbol(current, next)) {
                return false;
            }
            if(current == '(') rOpen++;
            else if(current == '[') sOpen++;
            else if(current == ')') rOpen--;
            else if(current == ']') sOpen--;
            if(rOpen < 0 || sOpen < 0) return false;
        }
        final char lastChar = s.charAt(s.length() - 1);
        if(lastChar == '(') rOpen++;
        else if(lastChar == '[') sOpen++;
        else if(lastChar == ')') rOpen--;
        else if(lastChar == ']') sOpen--;

        return s.length() > 1 && rOpen == 0 && sOpen == 0;
    }

    private static boolean isValidSymbol(char from, char to) {
        if(from == '(') {
            return to == ')' || to == '['  || to == '(' ;
        }
        else if(from == '[') {
            return to == ']' || to == '(' || to == '[';
        }
        return true;        
    }


public static void main(String[] args) {
        String testInput = "()[]";
        assert isBalanced(testInput) : "expected true";

        testInput = "([)]";
        assert isBalanced(testInput) == false : "expected false";

        testInput = "[([([()])])]";
        assert isBalanced(testInput) : "expected true";


        testInput = "([()([])()][()(())]())";
        assert isBalanced(testInput) : "expected true";

        testInput = "([()([])())[()(())]()) ";
        assert isBalanced(testInput) == false : "expected false";

    }